def gibbs_sampling(self, predict=dict(), given=dict(), n=10000, skip = 50):
bar = Bar('Sampling', max=n)
nodes = list(self.node.keys())
sample = self.random_sample(preset=given)
count = 0
sum = 0
for i in range(n):
last = None
bar.next()
node = None
last = node
while node is None or node in given.keys() or node == last:
node = nodes[randint(0,len(nodes)-1)]
parents = self.node[node]['parents']
if parents[0] is None:
sample[node] = self.sample(node)
else:
given = {key: sample[key] for key in parents}
sample[node] = self.sample(node, given=given)
if count == skip:
evidence = {key: sample[key] for key in predict.keys()}
if not predict == evidence:
continue
sum += 1
else:
count += 1
bar.finish()
return sum/(n-count)
def crawl(self,current=False):
"""
The main function - crawl the league and mine some data.
"""
logging.info('Starting crawl')
self.driver.get(self.league_link)
self.team_names = set([unidecode(thr.text) for thr in \
self.driver.find_element_by_class_name("stat-table").find_elements_by_class_name("team-link")])
self.driver.find_element(By.XPATH, '//*[@id="sub-navigation"]').find_element(By.PARTIAL_LINK_TEXT, 'Fixtures').click()
self.played_months = self.get_played_months()
self.load_previous_data(current)
prog_bar = ChargingBar('Progress of %s crawling:'%' '.join([self.league,str(self.year)]),max=sum([len(self.fixtures[month]) for month in self.played_months[self.played_months.index(self.start_month)::-1]]))
for month in self.played_months[self.played_months.index(self.start_month)::-1]:
for game in self.fixtures[month]:
logging.info('Starting to parse game')
if game:
self.parse_game(game)
prog_bar.next()
logging.info('Finished game, moving to the next one')
else:
logging.info('Finished month, saving to disk')
self.save_month(month)
if current:
pass
# DBHandler(args_parser.LEAGUE_NAME).update_db(self.all_teams_dict,str(self.year))
else: #we're done - we can save to the DB now
# if not current:
DBHandler(args_parser.LEAGUE_NAME).insert_to_db(self.all_teams_dict,str(self.year))
self.driver.quit()
prog_bar.finish()
def rejection_sample(self, predict=dict(), given=dict(), n=10000):
sum = 0
bar = Bar('Sampling', max=n)
for i in range(n):
bar.next()
sample = self.compute_sample()
evidence = {key: sample[key] for key in given.keys()}
if not given == evidence:
continue
evidence = {key: sample[key] for key in predict.keys()}
if not predict == evidence:
continue
sum += 1
bar.finish()
return sum/n
def likelihood_weighting(self, predict=dict(), given=dict(), n=10000):
num = den = 0
bar = Bar('Sampling', max=n)
for i in range(n):
bar.next()
sample = self.compute_sample(preset=predict)
for node in predict.keys():
parents = self.node[node]['parents']
given_pa = {key: sample[key] for key in parents}
weight = float(self.get_probability(node, evidence=given_pa, value=predict[node]))
evidence = {key: sample[key] for key in given.keys()}
if given == evidence:
num += weight
den += weight
bar.finish()
return num/den
# compute how many bytes to expect in last block nblocks = int(math.ceil(int(f['size'])/float(bsize))) slack = nblocks*int(bsize) - int(f['size']) actualbytes = int(bsize) - slack # read slack space tmp = open(tmpfile, "rb") bcount = 0 hidden="" try: byte = tmp.read(1) while byte != "": bcount += 1 if bcount > actualbytes and ord(byte) != 0x0: hidden+=byte byte = tmp.read(1) finally: tmp.close() os.remove(tmpfile) # use subdir to store result files # create file named after inode and write hidden bytes if hidden != "": if not os.path.exists(directory): os.makedirs(directory) f_out = open(directory+"/"+f["inode"], 'w') try: f_out.write(hidden) finally: f_out.close() bar.next() bar.finish()
def create_examples(self,year,lookback=15,current=False):
"""
This function creates all the examples for self.league, year.
The examples are created using the given lookback.
"""
def update_all_teams_dict(res,all_teams_dict,team,first):
for fix in sorted(res):
if fix == 1 and res[fix] == {}:
all_teams_dict[team][fix] = []
continue
if first:
all_teams_dict[team][fix] = [res[fix][k] for k in sorted(res[fix])]
else:
all_teams_dict[team][fix] += [res[fix][k] for k in sorted(res[fix])]
def relative_features(arr1,arr2,fn):
combined_list_all_1 = [value for (value,key) in zip(arr1,fn) if key.split("all_pos")>1 ]
combined_list_att_1 = [value for (value,key) in zip(arr1,fn) if key.split("att_pos")>1 ]
combined_list_def_1 = [value for (value,key) in zip(arr1,fn) if key.split("def_pos")>1 ]
combined_list_all_2 = [value for (value,key) in zip(arr2,fn) if key.split("all_pos")>1 ]
combined_list_att_2 = [value for (value,key) in zip(arr2,fn) if key.split("att_pos")>1 ]
combined_list_def_2 = [value for (value,key) in zip(arr2,fn) if key.split("def_pos")>1 ]
all_rel = [1 for (val1,val2) in zip (combined_list_all_1,combined_list_all_2) if val1 > val2]
att_rel = [1 for (val1,val2) in zip (combined_list_att_1,combined_list_att_2) if val1 > val2]
def_rel = [1 for (val1,val2) in zip (combined_list_def_1,combined_list_def_2) if val1 > val2]
return float(len(all_rel))/len(combined_list_all_1), float(len(att_rel))/len(combined_list_att_1), float(len(def_rel))/len(combined_list_def_1)
from features.features import Features
temp_DB = self.temp_DB
all_teams_names = [g['_id'] for g in temp_DB[self.league].aggregate([{"$match":{"Year":int(year)}},{"$group":{"_id":"$GName"}}])]
all_teams_dict = {name:{} for name in all_teams_names}
features = Features(temp_DB[self.league],year,self.league)
features_names = []
prog_bar = ChargingBar('Creating examples for %s-%s'%(self.league,year),max=len(all_teams_dict))
for team in all_teams_dict:
res_by_all, res_by_non_avg = features.create_features(team,lookback)
if not features_names: features_names = features.features_names
update_all_teams_dict(res_by_all, all_teams_dict, team, True)
update_all_teams_dict(res_by_non_avg, all_teams_dict, team, False)
prog_bar.next()
examples = []
tags = []
curr_examples = []
prog_bar.finish()
for team in all_teams_names:
for fix in sorted(all_teams_dict[team]):
if fix == 1 and all_teams_dict[team][fix]==[]:
continue
curr_game = temp_DB[self.league].find_one({"GName":team,"Fix":fix,"Year":int(year)})
if curr_game is None:
continue
if curr_game["HA"]=="home":
vs_curr_game = temp_DB[self.league].find_one({"GName":curr_game["VS"],"VS":team,"HA":"away","Year":int(year)})
try:
vs_curr_fix = vs_curr_game["Fix"]
except TypeError as e:
vs_curr_fix = fix+1
all_teams_dict[curr_game["VS"]][vs_curr_fix] = []
if all_teams_dict[curr_game["VS"]][vs_curr_fix] == []:
continue
rel_all, rel_att, rel_def = relative_features(all_teams_dict[team][fix], all_teams_dict[curr_game["VS"]][vs_curr_fix], features_names)
examples += [np.array(all_teams_dict[team][fix])-np.array(all_teams_dict[curr_game["VS"]][vs_curr_fix])]
examples[-1] = np.concatenate((examples[-1],[rel_all, rel_att, rel_def]))
temp_dict = {"Ex":examples[-1],"Fix":curr_game["Fix"],"Res":curr_game["Result"],"Home":team,"Away":curr_game["VS"],"League":self.league}
curr_examples += [temp_dict]
tags += [curr_game["Tag"]]
if not current:
return examples,tags
else:
return curr_examples,tags
while i < repeat:
cl = 0
sT = int(time.time())
eT = sT + testing
while int(time.time()) < eT:
vspBenchMakeHash()
operations += 10
cl += 1
for _ in range(i-len(cc)+1):
cc.append(None)
cc[i] = math.ceil(cl/division)
progressBar.next()
i += 1
progressBar.finish()
print(RSTF + ' > Benchmark completed! Calculating results ...\n')
cc.sort()
seconds = int(time.time()) - startTime
operations = int(operations / seconds)
proceed = ' (~' + str(operations) + 'h/sec) '
bResult = INVF + ' ' + str( cc[repeat-1] ) + ' BQ ' + RSTF + proceed + '\n'
print(' > Benchmark result: ' + bResult )
input(' Press any key to exit ... ')
def get_transactions(session, from_date, to_date):
"""
This method extracts the list of transactions from the given time-frame by calling the PayPal API.
The single query cannot span across 31 days. In case the request spawns across more than 31 days, we need to query the data month-by-month and aggregate the results.
:param session: Session to be used for the HTTP calls
:type session: OAuth2Session object
:return: list of transaction IDs
:rtype: list
"""
i = 0
result = {}
result[i] = [
" Date", "Time", "Name", "Status Code", "Currency", "Value",
"To Email Address", "Transaction ID", "Custom Number", "Quantity",
"Item Title", "Country Code"
]
query_start = datetime.strptime(from_date, "%Y-%m-%dT%H:%M:%S")
request_end = datetime.strptime(to_date, "%Y-%m-%dT%H:%M:%S")
if (request_end - query_start).days > (3 * 365):
complain("The query can span across 3 years maximum")
if (query_start - datetime(2016, 7, 1)).days < 0:
complain("The historical data is available only from July 2016")
query_count = int((request_end - query_start).days) / int(31)
bar = ChargingBar('1/3 Listing trx ',
max=query_count,
suffix='%(percent).1f%% - %(eta)ds remaining')
while (query_start < request_end):
# The query needs to be split into multiple-ones if it spans across more than 31 days
if (request_end - query_start).days > 31:
query_end = query_start + timedelta(days=31)
else:
query_end = request_end
page = 1
total_pages = 1
# The response might come in multiple pages and each needs to be queried separately
while (page <= total_pages):
params = (
("start_date",
query_start.strftime("%Y-%m-%dT%H:%M:%S") + TIMEZONE_OFFSET),
("end_date",
query_end.strftime("%Y-%m-%dT%H:%M:%S") + TIMEZONE_OFFSET),
("fields", "all"),
("page_size", "500"
), # The maximum amount of transaction IDs per page is 500
("page", str(page)))
response = session.get(TRX_URL, params=params)
check_status_code(response.status_code)
reply = json.loads(response.content)
total_pages = int(reply['total_pages'])
# Append the data to the resulting list
for trx in reply['transaction_details']:
# print('TRX DETAILS: '+ str(trx) + '\n')
trx_data = []
trx_data.append(
extract_value(
trx,
['transaction_info', 'transaction_initiation_date'
])[:10]) # Date
trx_data.append(
extract_value(
trx,
['transaction_info', 'transaction_initiation_date'
])[12:19]) # Time
trx_data.append(
extract_value(
trx,
['payer_info', 'payer_name', 'alternate_full_name'
])) # Name
trx_data.append(
extract_value(trx,
['transaction_info', 'transaction_status'
])) # Status Code
trx_data.append(
extract_value(trx, [
'transaction_info', 'transaction_amount',
'currency_code'
])) # Currency
trx_data.append(
extract_value(
trx,
['transaction_info', 'transaction_amount', 'value'
])) # Value
trx_data.append(
extract_value(
trx,
['payer_info', 'email_address'])) # To Email Address
trx_data.append(
extract_value(trx, ['transaction_info', 'transaction_id'
])) # Transaction ID
trx_data.append(
extract_value(
trx,
['transaction_info', 'custom_field'])) # Custom Number
count = 0.0
title = ""
for item in extract_value(trx, ['cart_info', 'item_details']):
count += float(extract_value(item, ['item_quantity']))
title += extract_value(item, ['item_name']) + "; "
trx_data.append(int(count)) # Quantity
trx_data.append(title) # Item Title
trx_data.append(
extract_value(trx,
['shipping_info', 'address', 'country_code'
])) # Country code
result[i] = trx_data
i += 1
page += 1
# In case another query is required, the start_time should be 1 second after the end_time of the previous query parameter
query_start = query_end + timedelta(seconds=1)
bar.next()
bar.finish()
return result
"""
from console import clear
# ————————————
# Progress bar
# ____________
from progress.bar import ChargingBar as Bar
from time import sleep
bar = Bar("Progress Bar", max=20)
for i in range(20):
sleep(0.1)
bar.next()
bar.finish()
clear()
# ———————————————
# Carriage return
# _______________
for i in range(20):
print(f"\r{i+1}", end="")
sleep(0.1)
clear()
# ——————
# Images
def slika(width, height, max_depth, camera, light, objects):
#width = 300*2
#height = 200*2
#max_depth = 3
'''
camera = np.array([0, 0, 1])
'''
ratio = float(width) / height
screen = (-1, 1 / ratio, 1, -1 / ratio) # left, top, right, bottom
'''
light = { 'position': np.array([5, 5, 5]), 'ambient': np.array([1, 1, 1]), 'diffuse': np.array([1, 1, 1]), 'specular': np.array([1, 1, 1]) }
objects = [
#{ 'type': 'ball', 'center': np.array([-0.2, 0, -1]), 'radius': 0.7, 'ambient': np.array([0.1, 0, 0]), 'diffuse': np.array([0.7, 0, 0]), 'specular': np.array([1, 1, 1]), 'shininess': 100, 'reflection': 0.5, 'n2': 1.52 },
#{ 'type': 'ball', 'center': np.array([0.1, -0.3, 0]), 'radius': 0.1, 'ambient': np.array([0.1, 0, 0.1]), 'diffuse': np.array([0.7, 0, 0.7]), 'specular': np.array([1, 1, 1]), 'shininess': 100, 'reflection': 0.5, 'n2': 1.52},
{'type': 'cylinder', 'center': np.array([-0.2, 0.2, -0.4]), 'radius': 0.5, 'height': 0.6, 'direction': np.array([0.2, 1, 0.2]), 'ambient': np.array([0.1, 0.1, 0.1]), 'diffuse': np.array([0.7, 0, 0.7]), 'specular': np.array([1, 1, 1]), 'shininess': 100, 'reflection': 0.5, 'n2': 1.52},
#{ 'type': 'ball', 'center': np.array([-0.3, 0, 0]), 'radius': 0.15, 'ambient': np.array([0, 0.1, 0]), 'diffuse': np.array([0, 0.6, 0]), 'specular': np.array([1, 1, 1]), 'shininess': 100, 'reflection': 0.5, 'n2': 1.52 },
{ 'type': 'plane', 'normal': np.array([0, 1, 0]), 'point': np.array([0, -1, 0]), 'ambient': np.array([0.1, 0.1, 0.1]), 'diffuse': np.array([0.6, 0.6, 0.6]), 'specular': np.array([1, 1, 1]), 'shininess': 100, 'reflection': 0.5, 'n2': 1},
{ 'type': 'cone', 'center': np.array([-0.3, 0.2, -0.8]), 'height': 0.5, 'direction': np.array([0,-2,1]), 'ambient': np.array([0.1, 0, 0]), 'diffuse': np.array([0.7, 0, 0]), 'specular': np.array([1, 1, 1]), 'shininess': 100, 'reflection': 0.5, 'n2': 1.52 }
]
'''
image = np.zeros((height, width, 3))
bar = ChargingBar('Processing', max=height)
for i, y in enumerate(np.linspace(screen[1], screen[3], height)):
for j, x in enumerate(np.linspace(screen[0], screen[2], width)):
# screen is on origin
pixel = np.array([x, y, 0])
origin = camera
direction = normalize(pixel - origin)
color = np.zeros((3))
reflection = 1
for k in range(max_depth):
# check for intersections
nearest_object, min_distance = nearest_intersected_object(
objects, origin, direction)
if nearest_object is None:
break
intersection = origin + min_distance * direction
if nearest_object['type'] == 'ball':
normal_to_surface = normalize(intersection -
nearest_object['center'])
elif nearest_object['type'] == 'cylinder':
#normal_to_surface = normalize(intersection - nearest_object['center'])
v1 = intersection - nearest_object['center']
normal_to_surface = normalize(
v1 -
(np.dot(v1, normalize(nearest_object['direction']))) *
normalize(nearest_object['direction']))
elif nearest_object['type'] == 'plane':
normal_to_surface = normalize(nearest_object['normal'])
elif nearest_object['type'] == 'cone':
#ker gre za stožec z enako višino kot maksimalnim radijem, se zadeve poenostavijo, saj je normala pod kotom 45 stopinj
h1 = normalize(nearest_object['direction'])
v1 = normalize(intersection - nearest_object['center'])
x1 = normalize(
np.sqrt(2) * nearest_object['height'] * v1 -
nearest_object['height'] * h1)
normal_to_surface = (np.sqrt(2) / 2) * x1 - (np.sqrt(2) /
2) * h1
shifted_point = intersection + 1e-5 * normal_to_surface
intersection_to_light = normalize(light['position'] -
shifted_point)
_, min_distance = nearest_intersected_object(
objects, shifted_point, intersection_to_light)
intersection_to_light_distance = np.linalg.norm(
light['position'] - intersection)
is_shadowed = min_distance < intersection_to_light_distance
if is_shadowed:
break
illumination = np.zeros((3))
# ambient
illumination += nearest_object['ambient'] * light['ambient']
# diffuse
illumination += nearest_object[
'diffuse'] * light['diffuse'] * np.dot(
intersection_to_light, normal_to_surface)
# specular
intersection_to_camera = normalize(camera - intersection)
H = normalize(intersection_to_light + intersection_to_camera)
illumination += nearest_object['specular'] * light[
'specular'] * np.dot(normal_to_surface,
H)**(nearest_object['shininess'] / 4)
# reflection
color += reflection * illumination
reflection *= nearest_object['reflection']
origin = shifted_point
direction = reflected(direction, normal_to_surface, 1.,
nearest_object['n2'])
image[i, j] = np.clip(color, 0, 1)
yield image
#v programu: trenutna_slika = slika().next()
bar.next()
bar.finish()
#plt.imshow(image)
#plt.show()
plt.imsave('imageOriginal.png', image)
