def __updateData():
data_url = app.config["DATA_URL"]
data_fetcher = DataFetcher(data_url)
try:
update_lock.acquire()
app.logger.info("Getting data from " + data_url)
global studies_text
studies_text = data_fetcher.get_studies_text()
app.logger.info("Finished fetching data from " + data_url)
global data_last_updated
data_last_updated = __now_string()
except (DataFetchException, requests.exceptions.RequestException) as e:
app.logger.info("Error fetching data from " + data_url +
"\nCannot update wiki data. Visualization data" +
"\nmay not be up to date or may be completely empty" +
"\nuntil the URL is successfully reachable."
"\nPlease ensure that the DATA_URL parameter is" +
" valid and restart the application." +
"\nMore detailed error:")
app.logger.error(e)
finally:
update_lock.release()
def __updateData():
data_url = app.config["DATA_URL"]
data_fetcher = DataFetcher(data_url)
try:
update_lock.acquire()
app.logger.info("Getting data from " + data_url)
global studies_text
studies_text = data_fetcher.get_studies_text()
app.logger.info("Finished fetching data from " + data_url)
global data_last_updated
data_last_updated = __now_string()
except (DataFetchException, requests.exceptions.RequestException) as e:
app.logger.info("Error fetching data from " + data_url +
"\nCannot update wiki data. Visualization data" +
"\nmay not be up to date or may be completely empty" +
"\nuntil the URL is successfully reachable."
"\nPlease ensure that the DATA_URL parameter is" +
" valid and restart the application." +
"\nMore detailed error:")
app.logger.error(e)
finally:
update_lock.release()
def initialize(self, uni):
"""docstring for __init__"""
self.arduino = Arduino()
self.arduino.start()
self.client = MongoClient('localhost', 27017)
self.uni = uni
self.mongo_gestures = self.client.affordance.gestures
self.mongo_envelopes = self.client.affordance.envelopes
self.mongo_users = self.client.affordance.users
self.mongo_tracking = self.client.affordance.tracking
self.mongo_studies = self.client.affordance.studies
self.list_obj = []
self.current_study_participant = None
self.data_fetcher = DataFetcher(self.client.affordance)
self.zap_speed = 1500
self.zap_strength = {"ems1": self.arduino.channels['ems1']['min_max'][0], "ems2": self.arduino.channels['ems2']['min_max'][0]}
self.zap_gestures = {"squeeze": False, "shake": False, "repel": False, "tap": False, "rotate": False, "ems1": False, "ems2": False, "ems3": False, "ems4": False}
#self.data_fetcher = DataFetcher(self.mongo_tracking, False) # Recorded data!
self.hand = Hand(self.data_fetcher, self.client.affordance, self.arduino)
self.uni.hand = self.hand
#screw_driver = AffordanceObjects(self.data_fetcher, self.uni, self.arduino, ["screwdriver_trackable"], "screwdriver", "Screwdriver")
#screw_brick = AffordanceObjects(self.data_fetcher, self.uni, self.arduino, ["screw_brick_trackable"], "screw_brick", "Screw brick")
#teapot = AffordanceObjects(self.data_fetcher, self.uni, self.arduino, ["teapot_trackable"], "teapot", "Teapot")
#spray_can = AffordanceObjects(self.data_fetcher, self.uni, self.arduino, ["spray_can_trackable"], "spray_can", "Spray Can")
#lamp = AffordanceObjects(self.data_fetcher, self.uni, self.arduino, ["hot_cup_trackable"], "hot_cup", "hot_cup")
#paint_brush = AffordanceObjects(self.data_fetcher, self.uni, self.arduino, ["paint_trackable"], "paint_brush", "Paint brush")
self.hand.start()
self.data_fetcher.register_trackers()
def do_inference(hostport, concurrency, testing_files, scaler):
"""
:param hostport: The host-port of the server
:param concurrency: The number of concurrent requests made to the server
:param testing_files: The files used for testing
:param scaler: The scaler to use to normalize features
:return: The inference error
"""
data_fetcher_test = DataFetcher(filepaths=testing_files,
is_train=False,
scaler=scaler,
unify_deltas=True,
unify_instances=False)
x_testing, y_testing = data_fetcher_test.fetch_all_classified()
num_tests = len(x_testing)
host, port = hostport.split(':')
channel = implementations.insecure_channel(host, int(port))
stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)
result_counter = _ResultCounter(num_tests, concurrency)
count = 0
batch_size = 10000
print("Running tests on " + str(num_tests) + " examples......")
while count < num_tests:
batch_end = min(count + batch_size, num_tests)
request = predict_pb2.PredictRequest()
request.model_spec.name = 'ec2pred_mlp'
request.inputs['input'].CopyFrom(
tf.contrib.util.make_tensor_proto(
x_testing[count:batch_end],
shape=x_testing[count:batch_end].shape,
dtype=tf.float32))
result_counter.throttle()
result_future = stub.Predict.future(request, 50000.0) # 5 seconds
result_future.add_done_callback(
_create_rpc_callback(
numpy.argmax(y_testing[count:batch_end], axis=1),
result_counter))
count += batch_size
return result_counter.get_error_rate()
def pressure():
# Extract the token from a request to the backend.
auth_token = request.headers.get('Authorization')
# Fetch the time series data from the IoT Time Series Service.
data = DataFetcher(auth_token, ASSET_ID, ASPECT).fetch_data('pressure')
# Create the line chart.
chart = DataPlotter().plot_line_chart(data[0], data[1], 'Air Pressure')
# Render the web page template.
return render_template('pressure.html', line_chart=Markup(chart))
def get_data():
"""HTTP endpoint to get data.
Retrieves downsampled data, that are within the given time range, from preprocessed raw files.
HTTP Args:
name: A string representing the fill name of the file user wish to
view. (example: DMM_res.csv)
strategy: A string representing the selected downsample strategy.
start (optional): An int representing the start of time span user wish to view.
end (optional): An int representing the end of time span user wish to view.
"""
name = request.args.get('name', type=str)
strategy = request.args.get('strategy', default='avg', type=str)
start = request.args.get('start', default=None, type=int)
end = request.args.get('end', default=None, type=int)
number = request.args.get(
'number', default=NUMBER_OF_RECORDS_PER_REQUEST, type=int)
if name is None:
warning('Empty file name.')
response = make_response('Empty file name')
return response, 400
if not strategy in STRATEGIES:
warning('Incorrect Strategy: %s', strategy)
response = make_response('Incorrect Strategy: {}'.format(strategy))
return response, 400
client = storage.Client()
fetcher = DataFetcher(name, PREPROCESS_DIR,
client.bucket(PREPROCESS_BUCKET))
if not fetcher.is_preprocessed():
response = make_response('Preprocessing incomplete.')
return response, 404
data, frequency_ratio = fetcher.fetch(
strategy, number, start, end)
response_data = {
'data': data,
'frequency_ratio': frequency_ratio
}
response = app.make_response(jsonify(response_data))
response.headers['Access-Control-Allow-Credentials'] = 'true'
return response
def __init__(self, sport_center_business_hours, fetch_period_seces=5, write_to_storage_period_secs=60, error_delay_secs=3):
self.io_loop = None
self.data_fetcher = DataFetcher()
sqlite_file_path = pathlib.Path(__file__).parent / 'data.sqlite'
print('database path', sqlite_file_path)
self.sql_connection = sqlite3.connect(str(sqlite_file_path))
self.fetch_period_secs = fetch_period_seces
self.error_delay_secs = error_delay_secs
self.write_to_storage_period_secs = timedelta(seconds=write_to_storage_period_secs)
self.business_hours = sport_center_business_hours
self.last_write_to_database_time = datetime.now()
self.data_cache = None
self.live_subject = Subject()
def train(date=None, days=1):
if date is None:
date = datetime.now()
raw_x_data, y_data = DataFetcher.fetch_data(date, days=days)
print(raw_x_data)
print(y_data)
raw_x_data = array(raw_x_data)
y_data = array(y_data)
x_data = transform_data(raw_x_data)
print(x_data)
model = baseline_model()
n_splits = 5
dataset = make_dataset(x_data, y_data, n_splits)
for X_train, y_train, X_test, y_test in tfe.Iterator(dataset):
model.fit(X_train, y_train, epochs=1000)
model.evaluate(X_test, y_test)
return model
class Handicap(object):
def __init__(self, host, key):
self.data_fetcher = DataFetcher(host, key)
def calculate(self, player_name, tour, year, tournament_id, slope):
differentials = []
for round in range(1, 6):
scorecard = json.load(self.data_fetcher.get_scorecards_per_round(tour, year, tournament_id, round))
print scorecard
if 'round' in scorecard:
for player in scorecard['round']['players']:
if player['first_name']+' '+player['last_name'] == player_name:
gross_score = 0
par = player['course']['par']
for score in player['scores']:
gross_score += score['strokes']
differential = self.handicap_differential(gross_score, int(slope))
differentials.append(differential)
handicap = self.handicap_index(differentials)
course_handicap = self.course_handicap(handicap, int(slope))
return handicap, course_handicap
def handicap_index(self, differentials):
count = 0
total = 0
for differential in differentials:
count += 1
total += differential
return (total/count)*0.96
def handicap_differential(self, gross_score, slope_rating):
differential = (85 - gross_score) * 113 / slope_rating
return differential
def course_handicap(self, index, slope):
course_handicap = ( index * slope )/113
return course_handicap
from data_fetcher import DataFetcher
# Read list of URLs
sites = []
with open('top-dk.csv') as in_file:
for line in in_file:
sites.append('http://www.' + line.split(',')[1].strip())
# Fetch sites
fetcher = DataFetcher(sites)
fetcher.fetch(stop=10)
def __init__(self, host, key):
self.data_fetcher = DataFetcher(host, key)
def test__binary_search_decreasing(self, numbers, value, expected):
"""Tests binary search with list of numbers in decreasing order."""
preprocess = DataFetcher('dummy', 'dummy')
assert preprocess._binary_search(numbers, value, True) == expected
#!/usr/bin/env python3
from datetime import datetime
from flask import Flask, request, jsonify
import dateparser
from flask import send_from_directory
from db_gateway import DBGateway
from data_fetcher import DataFetcher
from flask import Flask
from flask_cors import CORS
import sched, time
import logging
import threading
db_gateway = DBGateway()
data_fetcher = DataFetcher(db_gateway)
app = Flask(__name__)
CORS(app)
logging.basicConfig(level=logging.ERROR)
log = logging.getLogger(__name__)
@app.route('/map-data', methods=['GET'])
def get_map_data():
query_date_string = request.args.get('date')
if query_date_string:
query_date = dateparser.parse(query_date_string)
else:
query_date = datetime.today().utcnow()
from data_fetcher import DataFetcher
from recommendations import Recommendations
if __name__ == '__main__':
file = '/Users/aggrom/Desktop/MSDS/5_Data_mining/Assignment_1/friend-recommender/data/facebook_combined.txt'
percentages = []
algo_list = ['common_neighbors', 'jaccard', 'adamic_adar', 'cosine']
fetcher = DataFetcher(file)
graph = fetcher.network_dict
recommender = Recommendations(graph)
recommender.compute_the_number_users_with_the_same_first_and_different_10_recommendations(
)
percentages.append(
recommender.compute_similarity_percentage('common_neighbors',
'jaccard'))
percentages.append(
recommender.compute_similarity_percentage('common_neighbors',
'adamic_adar'))
percentages.append(
recommender.compute_similarity_percentage('jaccard', 'adamic_adar'))
percentages.append(
recommender.compute_similarity_percentage('cosine',
'common_neighbors'))
percentages.append(
recommender.compute_similarity_percentage('cosine', 'jaccard'))
percentages.append(
import pandas as pd
import riskparity
import outils
from data_fetcher import DataFetcher
from datetime import datetime, timedelta
import os
# TODO implement local serving of the css file
today = datetime.today()
db_file = 'stock_prices_eod.sqlite3'
data_path = os.path.join(db_file)
print(os.getcwd())
fetcher = DataFetcher(data_path)
rp_portfolio = [{'label': stock, 'value': stock} for stock in fetcher.tickers]
external_stylesheets = ['https://codepen.io/trooperandz/pen/YRpKjo.css']
# external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']
app = dash.Dash(__name__, external_stylesheets=external_stylesheets)
app.layout = html.Div(
className='grid-container',
children=[
html.Div(className='menu-icon',
children=[
html.I(className="fas fa-bars header__menu",
children='SC')
]),
html.Header(
class WebsocketHandler(websocket.WebSocketHandler):
def initialize(self, uni):
"""docstring for __init__"""
self.arduino = Arduino()
self.arduino.start()
self.client = MongoClient('localhost', 27017)
self.uni = uni
self.mongo_gestures = self.client.affordance.gestures
self.mongo_envelopes = self.client.affordance.envelopes
self.mongo_users = self.client.affordance.users
self.mongo_tracking = self.client.affordance.tracking
self.mongo_studies = self.client.affordance.studies
self.list_obj = []
self.current_study_participant = None
self.data_fetcher = DataFetcher(self.client.affordance)
self.zap_speed = 1500
self.zap_strength = {"ems1": self.arduino.channels['ems1']['min_max'][0], "ems2": self.arduino.channels['ems2']['min_max'][0]}
self.zap_gestures = {"squeeze": False, "shake": False, "repel": False, "tap": False, "rotate": False, "ems1": False, "ems2": False, "ems3": False, "ems4": False}
#self.data_fetcher = DataFetcher(self.mongo_tracking, False) # Recorded data!
self.hand = Hand(self.data_fetcher, self.client.affordance, self.arduino)
self.uni.hand = self.hand
#screw_driver = AffordanceObjects(self.data_fetcher, self.uni, self.arduino, ["screwdriver_trackable"], "screwdriver", "Screwdriver")
#screw_brick = AffordanceObjects(self.data_fetcher, self.uni, self.arduino, ["screw_brick_trackable"], "screw_brick", "Screw brick")
#teapot = AffordanceObjects(self.data_fetcher, self.uni, self.arduino, ["teapot_trackable"], "teapot", "Teapot")
#spray_can = AffordanceObjects(self.data_fetcher, self.uni, self.arduino, ["spray_can_trackable"], "spray_can", "Spray Can")
#lamp = AffordanceObjects(self.data_fetcher, self.uni, self.arduino, ["hot_cup_trackable"], "hot_cup", "hot_cup")
#paint_brush = AffordanceObjects(self.data_fetcher, self.uni, self.arduino, ["paint_trackable"], "paint_brush", "Paint brush")
self.hand.start()
self.data_fetcher.register_trackers()
#def ar_callback(self, data):
# self.write_message("channel_data" + ";".join(map(lambda x: x[:-1][1:], data.split("#")))[1:])
def collider_cb(self, list_obj):
"""Callback that sends the list of colliders to the web interface"""
self.list_obj = list_obj
self.write_message("colliders_init,True")
def open(self):
print("WebSocket opened")
self.write_message("init," + json.dumps(self.arduino.channels).replace(",", "§") + "," + bson.json_util.dumps(self.mongo_users.find({})).replace(",", "§"))
self.uni.ask_for_colliders(self.collider_cb)
def on_message(self, msg):
message = msg.split(",")
if message[0] == "run":
if message[1] == "true":
self.arduino.stop = False
self.data_fetcher.stop = False
self.arduino.open_all_channels()
print("run")
else:
self.data_fetcher.stop = True
self.arduino.stop = True
self.arduino.close_all_channels()
print("stop")
elif message[0] == "teapot":
if message[1] == "1":
self.hand.current_state = "none"
elif message[1] == "2":
self.hand.current_state = "hot"
elif message[0] == "muscle_control":
if message[2] == "true":
if message[1] == "ems1":
self.arduino.send_ems_strength({"ems1": self.zap_strength["ems1"]})
self.arduino.change_relay_state("ems1", True)
elif message[1] == "ems2":
self.arduino.send_ems_strength({"ems2": self.zap_strength["ems2"]})
self.arduino.change_relay_state("ems2", True)
elif message[1] == "ems3":
self.arduino.send_ems_strength({"ems3": self.zap_strength["ems1"]})
self.arduino.change_relay_state("ems3", True)
elif message[1] == "ems4":
self.arduino.send_value("h")
elif message[1] == "0":
if self.zap_strength["ems1"] > 4:
self.zap_strength["ems1"] -= 5
self.write_message("ems_strength,ems1," + str(self.zap_strength["ems1"]))
elif message[1] == "-":
self.zap_strength["ems1"] = 0
self.write_message("ems_strength,ems1," + str(self.zap_strength["ems1"]))
elif message[1] == "+":
if self.zap_strength["ems1"] < 96:
self.zap_strength["ems1"] += 5
self.write_message("ems_strength,ems1," + str(self.zap_strength["ems1"]))
elif message[1] == "p":
if self.zap_strength["ems2"] > 4:
self.zap_strength["ems2"] -= 5
self.write_message("ems_strength,ems2," + str(self.zap_strength["ems2"]))
elif message[1] == "[":
self.zap_strength["ems2"] = 0
self.write_message("ems_strength,ems2," + str(self.zap_strength["ems2"]))
elif message[1] == "]":
if self.zap_strength["ems2"] < 96:
self.zap_strength["ems2"] += 5
self.write_message("ems_strength,ems2," + str(self.zap_strength["ems2"]))
else:
if message[1] == "ems1":
self.arduino.change_relay_state("ems1", False)
elif message[1] == "ems2":
self.arduino.change_relay_state("ems2", False)
elif message[1] == "ems3":
self.arduino.change_relay_state("ems3", False)
elif message[1] == "ems4":
self.arduino.send_value("i")
elif message[0] == "colliders_init_cb":
user_id = {"user_id": ObjectId(message[1])}
gestures = bson.json_util.dumps(list(self.mongo_gestures.find(user_id, {"name": True}))).replace(",", "§")
envelopes = []
if len(self.list_obj) > 0:
for envelope in self.list_obj.split(","):
envelope_obj = self.mongo_envelopes.find({"user_id": ObjectId(message[1]), "name": envelope})
if envelope_obj.count() == 0:
self.mongo_envelopes.insert({
"name": envelope,
"gesture": "",
"gesture duration": 1000,
"allpoints": [],
"individual_points": [],
"user_id": ObjectId(message[1])
})
envelopes = self.mongo_envelopes.find({"user_id": ObjectId(message[1])})
envelopes = bson.json_util.dumps(envelopes).replace(",", "§")
self.write_message("colliders," + gestures + "," + envelopes)
elif message[0] == "study_start_trial":
self.mongo_studies.update({"_id": self.current_study_participant}, {
"$set": {
"trials." + message[1] + ".type": message[2],
"trials." + message[1] + ".optitrack_data": [],
"trials." + message[1] + ".touch_data": [],
"trials." + message[1] + ".timestamp": time.time()
}
})
self.data_fetcher.get_data_for_study(self.current_study_participant, message[1])
self.hand.get_data_for_study(self.current_study_participant, message[1])
if message[2][-6:] == "-noems":
self.arduino.study_no_ems = True
self.start_time = time.time()
elif message[0] == "study_end_trial":
fat_var = self.data_fetcher.done_with_study_data()
self.mongo_studies.update({"_id": self.current_study_participant}, {
"$set": {
"trials." + message[1] + ".time": time.time() - self.start_time,
"trials." + message[1] + ".optitrack_data": fat_var
}
})
self.hand.done_with_study_data()
self.arduino.study_no_ems = False
elif message[0] == "calibrate":
self.arduino.calibration(message)
elif message[0] == "load_study":
data = self.mongo_studies.find_one({"participant_id": message[1]})
self.current_study_participant = data["_id"]
self.mongo_studies.update({"_id": self.current_study_participant}, {"$set": {"name": message[2], "trials": {}}})
self.write_message("study_data," + json.dumps(data['order']).replace(",", "§"))
elif message[0] == "save_channels":
self.mongo_users.update({"_id": ObjectId(message[1])}, {"$set": {
"channels." + message[2]: {
"min": message[3],
"max": message[4]
}
}})
elif message[0] == "door":
if message[1] == "e":
self.hand.door_mode = "enter"
elif message[1] == "k":
self.hand.door_mode = "knock"
elif message[0] == "load_user":
self.hand.user = message[2]
gestures = list(self.mongo_gestures.find({"user_id": ObjectId(message[2])}, {"name": True}))
if not message[1] == message[2]:
user_gestures = list(self.mongo_gestures.find({"user_id": ObjectId(message[1])}, {"name": True}))
new_gestures = []
for gesture in gestures:
for gest in user_gestures:
if gest['name'] == gesture['name']:
gest['name'] = gest['name'] + " (modified)"
gesture = gest
new_gestures.append(gesture)
gestures = new_gestures
the_user = self.mongo_users.find_one({"_id": ObjectId(message[1])})
if the_user is not None:
for channel, val in the_user['channels'].items():
if channel in self.arduino.channels.keys():
self.arduino.channels[channel]['min_max'] = [int(val['min']), int(val['max'])]
users = bson.json_util.dumps(the_user).replace(",", "§")
gestures = bson.json_util.dumps(gestures).replace(",", "§")
self.write_message("user_info," + users + "," + gestures)
elif message[0] == "add_user":
user_id = self.mongo_users.insert({
"name": message[1],
"channels": {}
})
self.write_message("newly_added_user_id," + bson.json_util.dumps(self.mongo_users.find_one({"_id": ObjectId(user_id)})).replace(",", "§"))
elif message[0] == "envelope":
if message[1] == "save":
already_exists = 0
if message[8] == "true":
already_exists = self.mongo_envelopes.find({"_id": ObjectId(message[3]), "user_id": ObjectId(message[2])}).limit(1).count()
if already_exists == 1:
self.mongo_envelopes.update({"_id": ObjectId(message[3])}, {"$set": {
"gesture": ObjectId(message[4]),
"gesture duration": message[5],
"allpoints": json.loads(message[6].replace("§", ",")),
"individual_points": json.loads(message[7].replace("§", ",")),
}})
else:
if message[8] == "true":
name = self.mongo_envelopes.find({"_id": ObjectId(message[3])}).limit(0)[0]['name']
else:
name = message[3]
self.mongo_envelopes.insert({
"name": name,
"gesture": ObjectId(message[4]),
"gesture duration": message[5],
"allpoints": json.loads(message[6].replace("§", ",")),
"individual_points": json.loads(message[7].replace("§", ",")),
"user_id": ObjectId(message[2])
})
elif message[0] == "gesture":
if message[1] == "save":
already_exists = self.mongo_gestures.find({"_id": ObjectId(message[3]), "user_id": ObjectId(message[2])}).limit(1).count()
if already_exists == 1:
self.mongo_gestures.update({"_id": ObjectId(message[3])}, {"$set": {
"allpoints": json.loads(message[4].replace("§", ", ")),
"line_segments": json.loads(message[5].replace("§", ", "))
}})
else:
old_thing = self.mongo_gestures.find({"_id": ObjectId(message[3])}).limit(0)[0]
self.mongo_gestures.insert({
"name": old_thing['name'],
"allpoints": json.loads(message[4].replace("§", ", ")),
"line_segments": json.loads(message[5].replace("§", ", ")),
"user_id": ObjectId(message[2])
})
gestures = list(self.mongo_gestures.find({"user_id": ObjectId(message[2])}, {"name": True}))
self.write_message("reload_gestures," + bson.json_util.dumps(gestures).replace(",", "§"))
elif message[1] == "save_new":
self.mongo_gestures.insert({
"name": message[3],
"allpoints": json.loads(message[4].replace("§", ", ")),
"line_segments": json.loads(message[5].replace("§", ", ")),
"user_id": ObjectId(message[2])
})
gestures = list(self.mongo_gestures.find({"user_id": ObjectId(message[2])}, {"name": True}))
self.write_message("reload_gestures," + bson.json_util.dumps(gestures).replace(",", "§"))
elif message[1] == "get":
gesture = bson.json_util.dumps(list(self.mongo_gestures.find({"_id": ObjectId(message[2])}))[0]).replace(",", "§")
self.write_message("gesture_info," + gesture)
elif message[1] == "test":
duration = 1000
if message[3] and message[3].isdigit():
duration = int(message[3])
self.arduino.perform_gesture(json.loads(message[2].replace("§", ", ")), duration)
def on_close(self):
print("WebSocket closed")
def temperature():
auth_token = request.headers.get('Authorization')
data = DataFetcher(auth_token, ASSET_ID, ASPECT).fetch_data('temperature')
chart = DataPlotter().plot_line_chart(data[0], data[1], 'Temperature')
return render_template('temperature.html', line_chart=Markup(chart))