def evaluate(rec_output_type, user_list, rec_size, user_2_true, user_2_pred):
user_2_tp_and_fp_and_prec_and_ndcg = {}
for user in user_list:
if user == "user37" and rec_output_type == "Single":
"stop"
tp = 0.0
fp = rec_size
prec = 0.0
ndcg = 0.0
if user in user_2_true and user in user_2_pred:
y_true = list(unique_everseen(user_2_true[user]))
y_pred = list(unique_everseen(user_2_pred[user]))
tp = count_tp(rec_output_type, y_true, y_pred)
#fp = len(y_pred) - tp
# NOTE Assumed if pred_size< rec_size(k), the missing part is fp
fp = max(len(y_pred), rec_size) - tp
prec = calculatePrecision(tp, fp)
dcg = calculateDcg(rec_size, y_true, y_pred)
idcg = calculateIdealDcg(rec_size, y_true)
ndcg = dcg / idcg
user_2_tp_and_fp_and_prec_and_ndcg[user] = (tp, fp, prec, ndcg)
# // write precs of eah user
# printer.printPrecisionEvalResult(folder + outputPrecfileName, controlUserIdList, userPrecMap, userTPMap, userFPMap);
#
# // calculate precision for overall results
# Double precisionTotal = EvaluateCheckin2011DB.precision(truePosTotal, falsePosTotal);
# String str = "Overall -- tpTotal: " + truePosTotal
# + " fpTotal: " + falsePosTotal
# + " prec: " + precisionTotal;
# printer.printString(folder + outputPrecfileName, str);
return user_2_tp_and_fp_and_prec_and_ndcg
def combine(cls, obj, perms, limit=None):
ret = []
limit = limit and int(limit)
if isinstance(obj, iterables):
head = List
it = obj
else:
head = obj.__class__
it = obj.args
if limit:
for per in perms:
if not is_integer(per):
raise FunctionException("Subsets::int")
ret += [
head(*x)
for x in islice(combinations(it, int(per)), limit - len(ret))
]
if len(ret) >= limit:
break
else:
for per in perms:
if not is_integer(per):
raise FunctionException("Subsets::int")
ret += [head(*x) for x in combinations(it, int(per))]
return List(*unique_everseen(ret))
def step(self, actions):
self.players = [
limit_to_size(move(player, action, self.prob_right_direction),
self.grid_size)
for player, action in zip(self.players, actions)
]
states = self.__get_state()
is_at_goal = [
player == goal for player, goal in zip(self.players, self.goals)
]
reward_is_at_goal = [-1 if x == False else 20 for x in is_at_goal]
# detect a crash
dup = list(unique_everseen(duplicates(self.players)))
# if a player's position appears twice, add -20 to the current reward
reward_is_crash = [
-19 if (player in dup) else 0 for player in self.players
]
rewards = [a + b for a, b in zip(reward_is_at_goal, reward_is_crash)]
done = True in is_at_goal
if self.render_board:
print("{}".format(self.render()))
return (states, rewards, done)
def test_traverse():
"""To test the traverse implementation we call gc.collect() while instances
of all the C objects are still valid."""
acc = iteration_utilities.accumulate([])
app = iteration_utilities.applyfunc(lambda x: x, 1)
cha = iteration_utilities.chained(int, float)
cla = iteration_utilities.clamp([], 0, 1)
com = iteration_utilities.complement(int)
con = iteration_utilities.constant(1)
dee = iteration_utilities.deepflatten([])
dup = iteration_utilities.duplicates([])
fli = iteration_utilities.flip(int)
gro = iteration_utilities.grouper([], 2)
ine = iteration_utilities.intersperse([], 1)
iik = iteration_utilities.ItemIdxKey(10, 2)
ite = iteration_utilities.iter_except(int, TypeError)
mer = iteration_utilities.merge([])
nth = iteration_utilities.nth(1)
pac = iteration_utilities.packed(int)
par = iteration_utilities.partial(int, 10)
rep = iteration_utilities.replicate([], 3)
rou = iteration_utilities.roundrobin([])
see = iteration_utilities.Seen()
sid = iteration_utilities.sideeffects([], lambda x: x)
spl = iteration_utilities.split([], lambda x: True)
sta = iteration_utilities.starfilter(lambda x: True, [])
suc = iteration_utilities.successive([])
tab = iteration_utilities.tabulate(int)
une = iteration_utilities.unique_everseen([])
unj = iteration_utilities.unique_justseen([])
gc.collect()
def removeDuplicates(data):
from iteration_utilities import unique_everseen
permissionkeys = ["readLabels", "readUsers", "writeLabels", "writeUsers"]
for key in permissionkeys:
if data.get(key):
data[key] = list(unique_everseen(data.get(key)))
return data
def valid(string):
if 8 in string or 11 in string or 14 in string:
return False
if not any(starmap(straight, successive(string, 3))):
return False
pairs = unique_everseen(starfilter(eq, successive(string)))
return 1 < count_items(pairs)
def find_files(data_path, directory, search_for_mm):
files_array = []
for x in directory:
if search_for_mm:
name = file_name_result.search(x)
else:
name = file_name.search(x)
if name is not None and name not in files_array:
files_array.append(name[0])
files_array = list(unique_everseen(files_array))
files_array.sort()
return files_array
def _check_no_duplicate_object_ids(self):
"""
Checks if all object IDs are unique.
Raises:
ValueError: if duplicate IDs were found
"""
duplicate_obj_ids = list(unique_everseen(duplicates(self.object_ids)))
if duplicate_obj_ids:
raise PVConfigurationException(
f'User configuration entries contain duplicate'
f'object IDs: {duplicate_obj_ids}')
def test_uniqueeverseen_unhashable1():
assert list(unique_everseen([{
T(1): T(1)
}, {
T(2): T(2)
}, {
T(1): T(1)
}])) == [{
T(1): T(1)
}, {
T(2): T(2)
}]
def limite_z(self, position_coeur, position_sein):
List_z_coeur = []
List_z_Sein = []
List_tot_z = []
for i in position_coeur:
List_z_coeur.append(i[2])
for j in position_sein:
List_z_Sein.append(j[2])
List_z_coeur = list(unique_everseen(duplicates(List_z_coeur)))
List_z_Sein = list(unique_everseen(duplicates(List_z_Sein)))
slice_barycenter = int(
np.ceil((np.min(List_z_Sein) + np.max(List_z_Sein)) / 2))
List_tot_z = List_z_Sein + List_z_coeur
mini = np.min(List_tot_z)
maxi = np.max(List_tot_z)
# print("Mini = ", mini, "Maxi = ",maxi)
return mini, maxi, slice_barycenter
def pairs(actual, k):
all_act_pairs = []
if len(actual) >= k:
for pair in itertools.combinations(actual[:k], 2):
all_act_pairs.append(pair)
else:
if len(actual) == 1:
all_act_pairs.append((actual[0], actual[0]))
else:
for pair in itertools.combinations(actual, 2):
all_act_pairs.append(pair)
all_act_pairs = list(unique_everseen(all_act_pairs, key=set))
return float(len(all_act_pairs))
def test_uniqueeverseen_getter1():
t = unique_everseen([T(1), T([0, 0]), T(3)])
assert not t.seen
assert t.key is None
assert next(t) == T(1)
assert t.seen == Seen({T(1)})
assert t.key is None
assert next(t) == T([0, 0])
assert T(1) in t.seen
assert T([0, 0]) in t.seen
assert t.key is None
assert next(t) == T(3)
assert t.seen == Seen({T(1), T(3)}, [T([0, 0])])
assert t.key is None
def pairsu(actual, predicted, k):
actual = list(dict.fromkeys(actual))
predicted = list(dict.fromkeys(predicted))
all_act_pairs = []
for pair in itertools.combinations(actual[:k], 2):
all_act_pairs.append(pair)
all_pred_pairs = []
for pair in itertools.combinations(predicted[:k], 2):
all_pred_pairs.append(pair)
# The order is not important:
all_act_pairs = list(unique_everseen(all_act_pairs, key=set))
all_pred_pairs = list(unique_everseen(all_pred_pairs, key=set))
result = 0
for el in all_act_pairs:
if el in all_pred_pairs:
result = result + 1
return float(result)
def test_uniqueeverseen_getter2():
t = unique_everseen([T(1), T([0, 0]), T(3)],
iteration_utilities.return_identity)
assert not t.seen
assert t.key is iteration_utilities.return_identity
assert next(t) == T(1)
assert t.seen == Seen({T(1)})
assert t.key is iteration_utilities.return_identity
assert next(t) == T([0, 0])
assert T(1) in t.seen
assert T([0, 0]) in t.seen
assert t.key is iteration_utilities.return_identity
assert next(t) == T(3)
assert t.seen == Seen({T(1), T(3)}, [T([0, 0])])
assert t.key is iteration_utilities.return_identity
def permute(cls, obj, perms):
ret = []
if isinstance(obj, iterables):
head = List
it = obj
else:
head = obj.__class__
it = obj.args
for per in perms:
if not is_integer(per):
raise FunctionException("Permutations::int")
ret += [head(*x) for x in permutations(it, int(per))]
return List(*unique_everseen(ret))
def find_repeated_info(df):
global merged
global info_with_multiple_userid_ls
# find duplicate entries with many userid
df_colunm1_ls = df[df.columns[1]].to_list()
info_with_multiple_userid_ls = list(
unique_everseen(duplicates(df_colunm1_ls)))
# construct new df, to cut down on the number of rows
info_with_multiple_userid = pd.DataFrame(info_with_multiple_userid_ls,
columns=["info"])
info_with_multiple_userid.head()
merged = pd.merge(info_with_multiple_userid,
df,
how="left",
left_on="info",
right_on=df.columns[1]).drop(df.columns[1], axis=1)
def printOverallNetworkStatistic():
#Global Variables:
global tbs
global mfb
#Compute the total similarity average of all social troll bot networks used in the 2016 elections:
tsa = sum(tbs) / (len(tbs))
#Obtain the list of most frequent commonly used social troll bots used in the 2016 elections:
mfb = list(unique_everseen(duplicates(mfb)))
#Print Statements:
print(
"The average % of similarty of all politically social troll bot networks used in the 2016 elections = "
+ str(tsa) + "%")
print(
"The list of the most commonly used social troll bots used in the 2016 elections: "
+ str(mfb))
print(
"The no. of most commonly used social troll bots used in the 2016 elections: "
+ str(len(mfb)))
return True
def query(self, vector, radius=1, top_k=5):
res_indices = []
## Need to improve index calculations
indices = vector.dot(self.base_vector.T).reshape(self.num_tables,
-1) > 0
if radius == 0:
res_indices = indices.dot(2**np.arange(
self.n_vectors)) + np.arange(
self.num_tables) * 2**self.n_vectors
elif radius == 1:
clone_indices = indices.repeat(axis=0, repeats=self.n_vectors)
rel_indices = (np.arange(self.num_tables) *
2**self.n_vectors).repeat(axis=0,
repeats=self.n_vectors)
translate = np.tile(np.eye(self.n_vectors), (self.num_tables, 1))
res_indices = (np.abs(clone_indices - translate).dot(2**np.arange(
self.n_vectors)) + rel_indices).astype(int)
res_indices = np.concatenate([
res_indices,
indices.dot(2**np.arange(self.n_vectors)) +
np.arange(self.num_tables) * 2**self.n_vectors
])
start = time.time()
lst = self.hash_table[res_indices].tolist()
self.lookup_index_times.append(time.time() - start)
start = time.time()
res = list(unique_everseen(duplicates(flatten(lst))))
sim_scores = vector.dot(self.vectors[res].T)
max_sim_indices = sim_scores.argsort()[-top_k:][::-1]
max_sim_scores = sim_scores[max_sim_indices]
return [(self.names[res[i]], score)
for i, score in zip(max_sim_indices, max_sim_scores)]
def test_uniqueeverseen_empty1():
assert list(unique_everseen([])) == []
def test_uniqueeverseen_failure_setstate3():
_hf.iterator_setstate_empty_fail(unique_everseen(toT([1, 1])))
def test_uniqueeverseen_key2():
assert list(unique_everseen([T(1), T(1), T(-1)], abs)) == [T(1)]
def test_uniqueeverseen_failure1():
with pytest.raises(_hf.FailIter.EXC_TYP, match=_hf.FailIter.EXC_MSG):
unique_everseen(_hf.FailIter())
def test_uniqueeverseen_pickle1(protocol):
uqe = unique_everseen([T(1), T(2), T(1), T(2)])
assert next(uqe) == T(1)
x = pickle.dumps(uqe, protocol=protocol)
assert list(pickle.loads(x)) == [T(2)]
def test_uniqueeverseen_unhashable2():
assert list(unique_everseen([[T(1)], [T(2)], [T(1)]])) == [[T(1)], [T(2)]]
def test_uniqueeverseen_normal1():
assert list(unique_everseen([T(1), T(2), T(1)])) == [T(1), T(2)]
def test_uniqueeverseen_failure2():
with pytest.raises(TypeError):
list(unique_everseen([T(1), T(2), T(3), T('a')], abs))
def finaljson(request):
try:
engine = create_engine('postgresql://*****:*****@localhost:5432/postgres')
# l3.to_sql('legacytable3', engine, if_exists='append')
l1 = pd.read_sql_query('select * from "legacytable1"', con=engine)
l2 = pd.read_sql_query('select * from "legacytable2"', con=engine)
l3 = pd.read_sql_query('select * from "legacytable3"', con=engine)
merg = pd.merge(l1, l2, on="Emp#", how="outer")
merg1 = pd.merge(merg, l3, on="Emp#", how="outer")
merg1["Full Name"] = merg1["First Name"] + " " + merg1["Last Name"]
def digit_tracker(x):
d = str(x)
k = len(d)
spd = "0" * (11 - k) + d
return spd
merg1['uniqueid'] = merg1['Emp#'].apply(digit_tracker)
final = merg1[["uniqueid", "Full Name", "Skills", "Technology", "Tax ID", "City", "Country Code", "Phone", "Email"]]
final
# final.to_sql('new_table', engine, if_exists='append')
# final.to_csv("new_table.csv")
print("your new table is created and stored.")
mergedata1 = [
{"id": "03", "name": "ankit"},
{"id": "03", "address": "yyyy"},
{"id": "02", "phone": "3434343"},
{"id": "03", "phone": "23344"},
{"id": "01", "address": "rrrr"},
]
tablex = final.to_json(orient='records')
# code for reshape json and remove redundancy
plusdata=mergedata1
print(plusdata)
plusdata.sort(key=lambda item: item.get("id"))
print(plusdata)
for i in range(len(plusdata)-1):
if (plusdata[i]["id"] == plusdata[i + 1]["id"]):
plusdata[i].update(plusdata[i + 1])
for i in range(len(plusdata)-1):
if (plusdata[i]["id"] == plusdata[i + 1]["id"]):
plusdata[i].update(plusdata[i + 1])
plusdata[i + 1].update(plusdata[i])
fitdata=list(unique_everseen(plusdata, key=lambda item: frozenset(item.items())))
print(fitdata)
return render(request, 'dmac/product.html', {'tablex': tablex, 'fitdata': fitdata})
except:
mergedata1 = [
{"id": "03", "name": "ankit"},
{"id": "03", "address": "yyyy"},
{"id": "02", "phone": "3434343"},
{"id": "03", "phone": "23344"},
{"id": "01", "address": "rrrr"},
]
# tablex = final.to_json(orient='records')
# code for reshape json and remove redundancy
plusdata = mergedata1
print(plusdata)
plusdata.sort(key=lambda item: item.get("id"))
print(plusdata)
for i in range(len(plusdata) - 1):
if (plusdata[i]["id"] == plusdata[i + 1]["id"]):
plusdata[i].update(plusdata[i + 1])
for i in range(len(plusdata) - 1):
if (plusdata[i]["id"] == plusdata[i + 1]["id"]):
plusdata[i].update(plusdata[i + 1])
plusdata[i + 1].update(plusdata[i])
fitdata = list(unique_everseen(plusdata, key=lambda item: frozenset(item.items())))
print(fitdata)
return render(request, 'dmac/product.html', {'fitdata': fitdata})
def thankyou(request):
engine = create_engine('postgresql://*****:*****@localhost:5432/postgres')
# engine = create_engine('postgresql://*****:*****@localhost:5432/postgres')
df_aws2 = pd.read_sql_query("select relname from pg_class where relkind='r' and relname !~ '^(pg_|sql_)';",
con=engine)
tablename_list = list(df_aws2["relname"])
# # l3.to_sql('legacytable3', engine, if_exists='append')
l1 = pd.read_sql_query('select * from "saptable1" ', con=engine)
mkt = l1.iloc[0:, 1:]
data=mkt
# response = requests.get('http://127.0.0.1:8000/rest/')
# tables = pd.read_html(r.text)
# print('Tables found:', len(tables))
# print(tables)
# print(r.text)
# Parse the HTML pages
# from bs4 import BeautifulSoup
# soup = BeautifulSoup(response.text, 'html.parser')
# det = soup.find_all("table")
# tables = pd.read_html(response.text)
# print(tables)
# You can extract the page title as string as well
# Find all the h3 elements
# print(f"{tutorialpoints_page.find_all('h2')}")
# tags = tutorialpoints_page.find(
# lambda elm: elm.name == "h2" or elm.name == "h3" or elm.name == "h4" or elm.name == "h5" or elm.name == "h6")
# for sibling in tags.find_next_siblings():
# if sibling.name == "table":
# my_table = sibling
# df = pd.read_html(str(my_table))
# print(df)
if request.method == "POST":
tabl = request.POST.get('jsonData')
rules = request.POST.getlist('rules')
print(rules)
sit = json.loads(tabl)
frame = pd.DataFrame(sit)
frame = frame.iloc[0:, 1:]
plusdata1 = frame.to_json(orient='records')
plusdata2 = json.loads(plusdata1)
# git = {'lispr':sit}
# print(git)
def del_none(d):
"""
Delete keys with the value ``None`` in a dictionary, recursively.
This alters the input so you may wish to ``copy`` the dict first.
"""
# For Python 3, write `list(d.items())`; `d.items()` won’t work
# For Python 2, write `d.items()`; `d.iteritems()` won’t work
for key, value in list(d.items()):
if value == "":
del d[key]
elif isinstance(value, dict):
del_none(value)
return d # For convenience
listdf = []
for f in plusdata2:
listdf.append(del_none(f.copy()))
print(del_none(f.copy()))
# def del_none(f):
# for key, value in list(f.items()):
# if value is None:
# del f[key]
# elif isinstance(value, dict):
# del_none(value)
# lift.append(f)
# # return d
#
# del_none(f.copy())
print(listdf)
# print(frame)
# print(frame.info)
# print(plusdata)
# tablu = JSONParser().parse('jsonData')
# desc = request.POST.get('desc')
# table_name = json.loads(table_name.body)
# print(type(plusdata))
plusdata = listdf
plusdata.sort(key=lambda item: item.get("NAME"))
# print(plusdata)
for i in range(len(plusdata) - 1):
if (plusdata[i]["NAME"] == plusdata[i + 1]["NAME"]):
plusdata[i].update(plusdata[i + 1])
for i in range(len(plusdata) - 1):
if (plusdata[i]["NAME"] == plusdata[i + 1]["NAME"]):
plusdata[i].update(plusdata[i + 1])
plusdata[i + 1].update(plusdata[i])
fitdata = list(unique_everseen(plusdata, key=lambda item: frozenset(item.items())))
# print(fitdata)
l1 = pd.DataFrame(fitdata)
print(l1)
try:
if rules[0]=="name_spliter":
def string_split(x):
fd = x.split(" ")
return fd[0]
l1["first_name"] = l1["NAME"].apply(string_split)
def string_last(x):
try:
fd = x.split(" ")
return fd[1]
except:
pass
l1["last_name"] = l1["NAME"].apply(string_last)
table1 = l1.to_html()
elif rules[0]=="string resizer":
def digit_tracker(x):
d = str(x)
k = len(d)
if k <= 8:
spd = "0" * (8 - k) + d
return spd
else:
spd = d[0:8]
return spd
l1['PHONE_new'] = l1['PHONE'].apply(digit_tracker)
table1 = l1.to_html()
elif rules[0]=="first+last":
l1["merge_two_column"] = l1["NAME"]+" "+l1["WERKS"]
table1= l1.to_html()
else:
table1 = l1.to_html()
except:
l1.to_sql('mckinsolteam', engine, if_exists='append')
table1 = l1.to_html()
# print(rules[0])
return HttpResponse(table1)
def test_uniqueeverseen_unhashable3():
assert list(
unique_everseen([[T(1), T(1)], [T(1), T(2)], [T(1), T(3)]],
operator.itemgetter(0))) == [[T(1), T(1)]]
def test_uniqueeverseen_normal2():
# key=None is identical to no key
assert list(unique_everseen([T(1), T(2), T(1)], None)) == [T(1), T(2)]
