def __init__(self, color, previous, player, final, give_pie=False):
super().__init__()
self.logger = logging.getLogger("Question")
logging.basicConfig(
format='%(asctime)s - %(name)s: %(levelname)s - %(message)s',
level=logging.INFO)
self.final = final
self.color = color
self.previous = previous
self.player = player
self.handler = DatabaseHandler()
if color == 'black':
self.color = random.choice(['green', 'white', 'red', 'blue'])
self.question = self.handler.select_color_question(self.color)
self.answers = []
self.answers.append(self.question['correct'])
self.answers += self.question['incorrect']
random.shuffle(self.answers)
self.font = pygame.font.SysFont("Arial", 14)
self.selected_answer = ""
self.width, self.height = pygame.display.get_surface().get_size()
self.delay = 0
self.give_pie = give_pie
self.logger.info("Asking {0} a {1} question".format(
self.player.name, self.color))
def favorite_click(self, event):
if self.food.is_favorite() == "1":
self.canvas.itemconfigure(self.pic_created,
image=self.not_favorite)
DatabaseHandler.unfavorite(self.category, self.food)
self.favorites_tab.delete_frame(self.food)
for frame in self.expired_tab.get_frames():
if self.food == frame.get_food():
frame.get_canvas().itemconfigure(self.pic_created,
image=self.not_favorite)
for tab in self.application.get_tabs_list():
if tab.get_category() == self.category:
for frame in tab.get_frames():
if frame.get_food() == self.food:
frame.get_canvas().itemconfigure(
self.pic_created,
image=frame.get_not_favorite_pic())
else:
self.canvas.itemconfigure(self.pic_created, image=self.favorite)
DatabaseHandler.set_favorite(self.category, self.food)
new_frame = FoodFrame(self.favorites_tab.get_tab_frame(),
self.food, self.category, self.favorites_tab,
self.expired_tab, self.application, False)
self.favorites_tab.add_frame(new_frame)
for frame in self.expired_tab.get_frames():
if self.food == frame.get_food():
frame.get_canvas().itemconfigure(self.pic_created,
image=self.favorite)
for tab in self.application.get_tabs_list():
if tab.get_category() == self.category:
for frame in tab.get_frames():
if frame.get_food() == self.food:
frame.get_canvas().itemconfigure(
self.pic_created,
image=frame.get_favorite_pic())
def test_get_address_without_route_returns_address_when_routes_empty(self):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler.add_address(location=MapLocation(latitude=5, longitude=6, id=1))
address_generator = handler.get_address_generator()
self.assertEqual(MapLocation(latitude=5, longitude=6, id=1),
address_generator.next(),
"Only MapLocation in addresses was not returned")
def test_get_address_without_route_returns_with_correct_id(self):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler.add_address(MapLocation(latitude=2, longitude=2, id=222))
address_generator = handler.get_address_generator()
self.assertEqual(MapLocation(latitude=2, longitude=2, id=222),
address_generator.next(),
"MapLocation should return correct id")
def test_add_addresses_table_adds_table(self, mock_init_db):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler._add_addresses_table()
c = handler.conn.cursor()
c.execute("SELECT NAME FROM sqlite_master WHERE "
"TYPE='table' and NAME='addresses'")
self.assertTrue(c.fetchone(), "addresses table not created")
def test_get_address_without_route_returns_with_correct_id(self):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler.add_address(MapLocation(latitude=2, longitude=2, id=222))
address_generator = handler.get_address_generator()
self.assertEqual(MapLocation(latitude=2, longitude=2, id=222),
address_generator.next(),
"MapLocation should return correct id")
def test_get_address_without_route_returns_address_when_routes_empty(self):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler.add_address(
location=MapLocation(latitude=5, longitude=6, id=1))
address_generator = handler.get_address_generator()
self.assertEqual(MapLocation(latitude=5, longitude=6, id=1),
address_generator.next(),
"Only MapLocation in addresses was not returned")
def test_add_routes_table_adds_table(self,
mock_init_db):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler._add_routes_table()
c = handler.conn.cursor()
c.execute("SELECT NAME FROM sqlite_master WHERE "
"TYPE='table' AND NAME='routes'")
self.assertTrue(c.fetchone(), "routes table not created")
def test_construct_db_calls_add_stop_table(self):
handler = DatabaseHandler('unit_test_db.sqlite3')
mock_add_addresses = Mock()
handler._add_addresses_table = mock_add_addresses
mock_add_stops = Mock()
handler._add_stops_table = mock_add_stops
handler.initialize_db()
self.assertTrue(mock_add_stops.called,
"initialize_db did not call _add_stops_table")
def test_add_address_uses_MapLocation_id_if_nonzero(self,
mock_init_db):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler._add_addresses_table()
address_location = MapLocation(latitude=0.33, longitude=4, id=100)
handler.add_address(address_location)
c = handler.conn.cursor()
c.execute("SELECT * FROM addresses")
self.assertEqual(100, c.fetchone()[0])
def test_add_address_adds_to_address_table(self,
mock_init_db):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler._add_addresses_table()
handler.add_address(location=MapLocation(latitude=0.56, longitude=9.5))
c = handler.conn.cursor()
c.execute("SELECT latitude, longitude FROM addresses")
row = c.fetchone()
self.assertEqual((0.56, 9.5), row)
def test_add_stop_uses_MapLocation_id_if_nonzero(self,
mock_init_db):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler._add_stops_table()
stop_location = MapLocation(latitude=0.48, longitude=179, id=888)
handler.add_stop(stop_location)
c = handler.conn.cursor()
c.execute("SELECT * FROM stops")
self.assertEqual(888, c.fetchone()[0])
def initDb(pathDb):
db = DatabaseHandler(u'qt_sql_default_connection', pathDb)
queries = [
('CREATE TABLE folders (name TEXT PRIMARY KEY, os TEXT, computerName TEXT)', None),
('CREATE TABLE files (hash TEXT, absPath TEXT, name TEXT, path TEXT, size NUM, searchTag NUM)', None),
('CREATE TABLE tags (searchTag NUM UNIQUE, timestamp NUM, computerName TEXT)', None),
('CREATE TABLE lastSearch (searchTag UNIQUE, lastPercent NUM, lastPath TEXT)', None)
]
db.processQueries(queries)
return
def init_game(self):
"""
Initialize the game
"""
self.logger.info("Initializing Database")
self.database = DatabaseHandler()
self.questionHandler = QuestionHandler()
self.dice = Dice(6)
self.logger.info("Polishing Dice")
self.logger.info("Starting Game - Welcome to trivial purfuit")
def test_add_stop_adds_to_stops_table(self,
mock_init_db):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler._add_stops_table()
map_location = MapLocation(latitude=-0.55, longitude=80)
handler.add_stop(location=map_location)
c = handler.conn.cursor()
c.execute("SELECT latitude, longitude FROM stops")
row = c.fetchone()
self.assertEqual((-.55, 80), row)
def parse_coords(lat, lon, db):
conn = DatabaseHandler.get_connection()
shape = (list(DatabaseHandler.get_records(conn, lat, lon, db)))[0]
# shape = 'MULTIPOLYGON (((211576.394869812 178026.964925658, 211562.853621803 178012.17132565, 211527.982069783 178045.20825367, 211536.800245792 178055.76134168, 211537.869941786 178057.041469678, 211540.061621793 178059.66444568, 211540.091253787 178059.699901681, 211540.138741791 178059.65714968, 211540.305141792 178059.507133681, 211576.394869812 178026.964925658)))'
p = shapely.wkt.loads(shape)
conn.close()
shp_file = find_containing_shp(lat, lon, "DSM")
tiff_file = extract_corresponding_tiff(shp_file)
analyse_tiff(tiff_file, lat, lon, p)
def test_handler_add_stops_from_file_inserts_one_record(self,
mock_init_db):
handler = DatabaseHandler('unit_test_db.sqlite3')
with open('test_file.csv', 'w') as f:
f.write('latitude,longitude\n')
f.write('15.35,-1.5\n')
handler.add_stops_from_file('test_file.csv')
c = handler.conn.cursor()
c.execute("SELECT latitude, longitude FROM stops")
row = c.fetchone()
self.assertEqual((15.35, -1.5), row)
def test_get_address_without_route_generator_new_address_second_time(self):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler.add_address(location=MapLocation(latitude=1, longitude=1))
handler.add_address(location=MapLocation(latitude=2, longitude=2))
address_generator = handler.get_address_generator()
self.assertEqual(MapLocation(latitude=1, longitude=1, id=1),
address_generator.next(),
"first returned MapLocation was not correct")
self.assertEqual(MapLocation(latitude=2, longitude=2, id=2),
address_generator.next(),
"second returned MapLocation was not correct")
def delete_click(self, event):
DatabaseHandler.delete_food(self.category, self.food)
if self.food.is_favorite() == "1":
self.favorites_tab.delete_frame(self.food)
if self.food.already_expired():
self.expired_tab.delete_frame(self.food)
for tab in self.application.get_tabs_list():
if tab.get_category() == self.category:
for frame in tab.get_frames():
if frame.get_food() == self.food:
frame.sub_master.destroy()
def test_get_address_without_route_generator_new_address_second_time(self):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler.add_address(location=MapLocation(latitude=1, longitude=1))
handler.add_address(location=MapLocation(latitude=2, longitude=2))
address_generator = handler.get_address_generator()
self.assertEqual(MapLocation(latitude=1, longitude=1, id=1),
address_generator.next(),
"first returned MapLocation was not correct")
self.assertEqual(MapLocation(latitude=2, longitude=2, id=2),
address_generator.next(),
"second returned MapLocation was not correct")
def test_handler_add_stops_from_file_inserts_one_record(
self, mock_init_db):
handler = DatabaseHandler('unit_test_db.sqlite3')
with open('test_file.csv', 'w') as f:
f.write('latitude,longitude\n')
f.write('15.35,-1.5\n')
handler.add_stops_from_file('test_file.csv')
c = handler.conn.cursor()
c.execute("SELECT latitude, longitude FROM stops")
row = c.fetchone()
self.assertEqual((15.35, -1.5), row)
def test_get_all_stops_returns_list_of_MapLocations(self, mock_init_db):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler._add_stops_table()
handler.add_stop(MapLocation(latitude=5, longitude=6))
handler.add_stop(MapLocation(latitude=3, longitude=-5))
stops = handler.get_all_stops()
self.assertEqual((5, 6), (stops[0].latitude, stops[0].longitude))
self.assertEqual((3, -5), (stops[1].latitude, stops[1].longitude))
def create_index(self):
self.word_docs = {}
self.inverted_index = {}
db = DatabaseHandler()
posts = db.get_posts()
total_posts = len(posts)
for doc_id in posts:
self.pos_tagger(posts[doc_id].text, doc_id)
self.compute_inverted_index(total_posts)
def test_output_routes_calls_correct_function_when_closest_stop_true(self):
handler = DatabaseHandler(full=False)
handler.routes_dataframe = Mock()
handler.routes_dataframe_closest_stops = Mock()
handler.output_routes(file_path="test_file.csv",
closest_stops_only=True)
handler.routes_dataframe_closest_stops.assert_called_once_with()
self.assertEqual(0, handler.routes_dataframe.call_count,
"routes_dataframe should not be called")
def test_add_route_adds_route(self):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler.add_address(location=MapLocation(latitude=5, longitude=5))
handler.add_stop(location=MapLocation(latitude=2, longitude=2))
handler.add_route(address=1, stop=1, distance=10, time=20)
c = handler.conn.cursor()
c.execute("SELECT * FROM routes")
self.assertEqual((1, 1, 1, 10, 20), c.fetchone())
def test_get_all_stops_returns_list_of_MapLocations(self,
mock_init_db):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler._add_stops_table()
handler.add_stop(MapLocation(latitude=5, longitude=6))
handler.add_stop(MapLocation(latitude=3, longitude=-5))
stops = handler.get_all_stops()
self.assertEqual((5, 6), (stops[0].latitude, stops[0].longitude))
self.assertEqual((3, -5), (stops[1].latitude, stops[1].longitude))
def handle_socket(conn):
try:
data = recv_msg(conn)
if (data != None):
data = data.decode('utf-8')
print("[LOG] Receive Data")
niceJson = json.loads(data)
Database = DatabaseHandler(DATABASE_IP, DATABASE_PORT,
"smartSystem")
Database.insertBulkDocument("sensor", niceJson)
except (socket.error):
conn.close()
print("[Warn] Connection Closed by Peer")
except:
print("[Warn] Unexpected error:", sys.exc_info())
conn.close()
def test_add_route_adds_route(self):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler.add_address(location=MapLocation(latitude=5, longitude=5))
handler.add_stop(location=MapLocation(latitude=2, longitude=2))
handler.add_route(address=1, stop=1, distance=10, time=20)
c = handler.conn.cursor()
c.execute("SELECT * FROM routes")
self.assertEqual((1, 1, 1, 10, 20), c.fetchone())
def printSuggestions(outputFilePath, filmInfoDatabasePath):
filmInfoDatabase = db.loadFilmInfoDatabase(filmInfoDatabasePath)
filmInfoDatabaseScoreList = ms.getFilmObjects(outputFilePath, filmInfoDatabase, True)
i = 0
# Underline the links and print them out along with the title in parentheses
for film in filmInfoDatabaseScoreList:
i += 1
print(str(i) + ':\t' + '\033[4m' + 'https://www.imdb.com/title/tt' + str(film['id']) + '\033[0m' + ' (' + film['n'] + ')')
def main():
os.chdir(os.path.dirname(os.path.realpath(__file__)))
config_file = parse_args()
conf = load_config(config_file)
init_logging(conf['DEFAULT']['LogPath'])
dh = DatabaseHandler(
host=conf['DATABASE']['Host'],
user=conf['DATABASE']['User'],
password=conf['DATABASE']['Password'],
db_name=conf['DATABASE']['DB']
)
np = dh.readNewsProvider()
crawler.get_articles_from_news_providers(np, dh)
dh.close()
def test_routes_dataframe_has_correct_values(self):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler.initialize_db()
handler.add_address(MapLocation(latitude=11, longitude=50, id=11))
handler.add_stop(MapLocation(latitude=-10, longitude=3, id=800))
handler.add_route(address=11, stop=800, distance=10000, time=50000)
df = handler.routes_dataframe()
self.assertEqual(1, df.shape[0], "only one row should be output")
self.assertEqual(11, df.ix[0, 'address_latitude'],
'incorrect address latitude output')
self.assertEqual(50, df.ix[0, 'address_longitude'],
'incorrect address longitude output')
self.assertEqual(-10, df.ix[0, 'stop_latitude'],
'incorrect stop latitude output')
self.assertEqual(3, df.ix[0, 'stop_longitude'],
'incorrect stop longitude output')
self.assertEqual(10000, df.ix[0, 'distance'],
'incorrect distance output')
self.assertEqual(50000, df.ix[0, 'time'],
'incorrect time output')
def __init__(self):
print("STARTING...")
print("startup process, will be notified when startup is successful")
print(
"startup will open connection to mongodb, if not successful, server will crash"
)
print("\n")
logging.debug("starting mother, setting name, address, family to '' ")
logging.debug("initializing database handler")
self.databasehandler = DatabaseHandler()
logging.debug("initializing self.devices to empty list")
self.devices = []
def test_construct_db_calls_add_route_table(self):
handler = DatabaseHandler('unit_test_db.sqlite3')
mock_add_addresses = Mock()
handler._add_addresses_table = mock_add_addresses
mock_add_stops = Mock()
handler._add_stops_table = mock_add_stops
mock_add_routes = Mock()
handler._add_routes_table = mock_add_routes
handler.initialize_db()
self.assertTrue(mock_add_routes.called,
"initialize_db did not call _add_routes_table")
def test_add_stop_uses_MapLocation_id_if_nonzero(self, mock_init_db):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler._add_stops_table()
stop_location = MapLocation(latitude=0.48, longitude=179, id=888)
handler.add_stop(stop_location)
c = handler.conn.cursor()
c.execute("SELECT * FROM stops")
self.assertEqual(888, c.fetchone()[0])
def test_add_address_uses_MapLocation_id_if_nonzero(self, mock_init_db):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler._add_addresses_table()
address_location = MapLocation(latitude=0.33, longitude=4, id=100)
handler.add_address(address_location)
c = handler.conn.cursor()
c.execute("SELECT * FROM addresses")
self.assertEqual(100, c.fetchone()[0])
def test_add_address_adds_to_address_table(self, mock_init_db):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler._add_addresses_table()
handler.add_address(location=MapLocation(latitude=0.56, longitude=9.5))
c = handler.conn.cursor()
c.execute("SELECT latitude, longitude FROM addresses")
row = c.fetchone()
self.assertEqual((0.56, 9.5), row)
def __init__(self,
circuit_params=None,
general_params=None,
database_path=None,
settings=None):
if settings is None:
self.settings = Settings(defaults.SETTINGS)
if not os.path.isdir(self.settings.general.scratch_dir):
os.mkdir(self.settings.general.scratch_dir)
self.circuit_params = circuit_params
self.general_params = general_params
self.task_sets = []
self.db_handler = DatabaseHandler(self.settings.databases,
database_path)
self.circuit_submitter = CircuitSubmitter(self.settings.general,
self.general_params)
self.circuit_validator = CircuitValidator()
self.circuit_critic = CircuitCritic(self.circuit_params)
self.circuit_designer = CircuitDesigner(self.settings.general,
self.circuit_params)
class Game():
"""
Game class
"""
def __init__(self):
self.logger = logging.getLogger("Game")
logging.basicConfig(
format='%(asctime)s - %(name)s: %(levelname)s - %(message)s',
level=logging.INFO)
self.colors = ['red', 'white']
self.init_game()
def init_game(self):
"""
Initialize the game
"""
self.logger.info("Initializing Database")
self.database = DatabaseHandler()
self.questionHandler = QuestionHandler()
self.dice = Dice(6)
self.logger.info("Polishing Dice")
self.logger.info("Starting Game - Welcome to trivial purfuit")
def run(self):
"""
Run loop for the game
Raises:
- sys.exit on keyboard interrupt
"""
try:
while (True):
# simulate roll
roll = self.dice.roll()
self.logger.info("You rolled a {0}".format(roll))
# simulate moving the piece
self.logger.info("Moving your piece...")
# simulate landing on a color
color = random.choice(self.colors)
self.logger.info("You landed on {0}".format(color))
self.logger.debug("Selecting {0} question".format(color))
# get a question from the color type
question = self.database.select_color_question(color)
# get an answer from the user
answer = self.questionHandler.query_user(question)
# evaluate the answer
self.questionHandler.evaluate(question, answer)
time.sleep(1)
except KeyboardInterrupt:
sys.exit()
def test_add_stop_adds_to_stops_table(self, mock_init_db):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler._add_stops_table()
map_location = MapLocation(latitude=-0.55, longitude=80)
handler.add_stop(location=map_location)
c = handler.conn.cursor()
c.execute("SELECT latitude, longitude FROM stops")
row = c.fetchone()
self.assertEqual((-.55, 80), row)
def test_get_address_without_route_returns_address_without_route(self):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler.add_address(location=MapLocation(latitude=1, longitude=2))
handler.add_address(location=MapLocation(latitude=3, longitude=4))
handler.add_stop(location=MapLocation(latitude=0, longitude=0))
c = handler.conn.cursor()
c.execute("INSERT INTO routes (address_id, stop_id, distance, time)"
"VALUES (1, 1, 1, 1)")
c.close()
address_generator = handler.get_address_generator()
self.assertEqual(MapLocation(latitude=3, longitude=4, id=2),
address_generator.next(),
"the MapLocation without route was not returned")
def clean(self):
used = False
for tab in self.application.tabs_list:
if tab.get_category().get_name() == self.entry.get():
used = True
if used:
window = InvalidInputWindow(self.master, self.application,
"This category already exists.")
else:
if self.valid_input_checker(self.entry.get()):
self.value = self.entry.get()
DatabaseHandler.create_category_database(self.entry.get())
new_tab = Tab(self.application.get_notebook(),
Category(self.entry.get()), self.application,
self.favorites_tab, self.expired_tab)
new_tab.populate_tab()
self.application.tabs_list.append(new_tab)
self.top.destroy()
else:
window = InvalidInputWindow(
self.master, self.application,
"Use only capital and lowercase letters,\nnumbers or apostrophe, please."
)
def test_select_color_question(self, random):
with patch('builtins.open',
mock_open(read_data="a,b,\"c, d, e\",blue")) as mock_file:
handler = DatabaseHandler()
self.assertEqual(handler.select_color_question('red'), "something")
self.assertEqual(handler.select_color_question('blue'),
"something")
self.assertEqual(handler.select_color_question('green'),
"something")
self.assertEqual(handler.select_color_question('white'),
"something")
def test_output_routes_calls_correct_function_when_closest_stop_true(self):
handler = DatabaseHandler(full=False)
handler.routes_dataframe = Mock()
handler.routes_dataframe_closest_stops = Mock()
handler.output_routes(file_path="test_file.csv",
closest_stops_only=True)
handler.routes_dataframe_closest_stops.assert_called_once_with()
self.assertEqual(0, handler.routes_dataframe.call_count,
"routes_dataframe should not be called")
def __init__(self, host, port, db):
"""
Initialises the service.
@type host: string
@param host: the IP to listen on.
@type port: integer
@param port: the port to listen on.
@type db: string
@param db: path to the SQLite database file.
"""
self.db = DatabaseHandler(db)
self.host = host
self.port = port
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,1)
self.sock.bind((host,port))
self.sock.listen(1)
self.modules = self.db.getModulePortsAndNames()
def test_get_address_without_route_returns_address_without_route(self):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler.add_address(location=MapLocation(latitude=1, longitude=2))
handler.add_address(location=MapLocation(latitude=3, longitude=4))
handler.add_stop(location=MapLocation(latitude=0, longitude=0))
c = handler.conn.cursor()
c.execute("INSERT INTO routes (address_id, stop_id, distance, time)"
"VALUES (1, 1, 1, 1)")
c.close()
address_generator = handler.get_address_generator()
self.assertEqual(MapLocation(latitude=3, longitude=4, id=2),
address_generator.next(),
"the MapLocation without route was not returned")
def main(self, MAX_POSTS=-1):
# Load and fit data set
print "Fitting data set"
tfs = self.fit_dataset()
# Take MAX_POSTS posts to classify from database
posts = DatabaseHandler().get_posts(MAX_POSTS).values()
for post in posts:
# Creating vector for new document
vec = self.vectorize_text(post.text)
# Calculate cosine similarity
sim = cosine_similarity(tfs, vec).flatten()
# Find top matching category
best_match_doc = sim.argmax()
print "\nUrl : %s matches best:" % post.url
print "Category %s with %.5f sim" % (
self.get_filename(best_match_doc), sim[best_match_doc])
def getTopSuggestedFilms(likedFilmsPath, filmInfoDatabasePath, maxCount):
filmInfoDatabase = db.loadFilmInfoDatabase(filmInfoDatabasePath)
likedFilms = MovieSuggester.getFilmObjects(likedFilmsPath,
filmInfoDatabase, False)
filmInfoDatabaseScoreList = []
for dbFilm in filmInfoDatabase:
score = MovieSuggester.__scoreFilm(dbFilm, likedFilms)
m = MovieElement(dbFilm['n'], dbFilm['id'], score)
filmInfoDatabaseScoreList.append(m)
filmInfoDatabaseScoreList.sort(key=lambda x: x.score, reverse=True)
filmInfoDatabaseScoreList = filmInfoDatabaseScoreList[:maxCount]
suggestedFilms = []
for movieObject in filmInfoDatabaseScoreList:
suggestedFilms.append(movieObject.id)
return suggestedFilms
def __init__(self, con_path):
self.__db_handler = DatabaseHandler(con_path)
def test_routes_dataframe_closest_stops_returns_for_many_addresses(self):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler.initialize_db()
handler.add_address(MapLocation(latitude=1, longitude=1, id=1))
handler.add_address(MapLocation(latitude=11, longitude=10, id=11))
handler.add_stop(MapLocation(latitude=2, longitude=2, id=2))
handler.add_stop(MapLocation(latitude=12, longitude=12, id=12))
handler.add_route(address=1, stop=2, distance=1, time=1)
handler.add_route(address=1, stop=12, distance=11, time=11)
handler.add_route(address=11, stop=2, distance=9, time=9)
handler.add_route(address=11, stop=12, distance=1, time=1)
df = handler.routes_dataframe_closest_stops()
self.assertEqual(2, df.shape[0], "should be 2 output rows since "
"there are 2 addresses with routes")
self.assertEqual(1, df.ix[0, 'distance'],
"distance for the first row should be 1 "
"since that is the shortest route distance")
def test_routes_dataframe_closest_stops_returns_closest_stop(self):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler.initialize_db()
handler.add_address(MapLocation(latitude=1, longitude=1, id=1))
handler.add_stop(MapLocation(latitude=2, longitude=2, id=2))
handler.add_stop(MapLocation(latitude=3, longitude=3, id=3))
handler.add_route(address=1, stop=2, distance=100, time=100)
handler.add_route(address=1, stop=3, distance=50, time=50)
df = handler.routes_dataframe_closest_stops()
self.assertEqual(1, df.shape[0], "should be 1 output row")
self.assertEqual(1, df.ix[0, 'address_latitude'],
'address should have latitude of 1')
self.assertEqual(1, df.ix[0, 'address_longitude'],
'address should have longitude of 1')
self.assertEqual(3, df.ix[0, 'stop_latitude'],
'the stop with latitude of 3 should have been '
'returned since it had the smallest distance')
self.assertEqual(3, df.ix[0, 'stop_longitude'],
'the stop with longitude of 3 should have been '
'returned since it had the smallest distance')
self.assertEqual(50, df.ix[0, 'distance'],
"the route with the lowest distance should be"
"returned in the output dataframe")
self.assertEqual(50, df.ix[0, 'time'],
"the route the with lowest distance's time should be"
"returned in the output dataframe")
def test_routes_dataframe_only_grabs_routes_no_dangling_locations(self):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler.initialize_db()
handler.add_address(MapLocation(latitude=1, longitude=1, id=1))
handler.add_address(MapLocation(latitude=2, longitude=2, id=2))
handler.add_address(MapLocation(latitude=3, longitude=3, id=3))
handler.add_stop(MapLocation(latitude=11, longitude=11, id=11))
handler.add_stop(MapLocation(latitude=12, longitude=12, id=12))
handler.add_stop(MapLocation(latitude=13, longitude=13, id=13))
handler.add_route(address=1, stop=11, distance=100, time=1000)
handler.add_route(address=3, stop=13, distance=100, time=1000)
df = handler.routes_dataframe()
self.assertEqual(2, df.shape[0], "should be 2 output rows")
self.assertEqual(1, df.ix[0, 'address_latitude'],
'incorrect address latitude output')
self.assertEqual(3, df.ix[1, 'address_latitude'],
'incorrect address latitude output')
self.assertEqual(11, df.ix[0, 'stop_latitude'],
'incorrect stop latitude output')
self.assertEqual(13, df.ix[1, 'stop_latitude'],
'incorrect stop latitude output')
class ManualSanityCheckerService:
def __init__(self, host, port, db):
"""
Initialises the service.
@type host: string
@param host: the IP to listen on.
@type port: integer
@param port: the port to listen on.
@type db: string
@param db: path to the SQLite database file.
"""
self.db = DatabaseHandler(db)
self.host = host
self.port = port
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,1)
self.sock.bind((host,port))
self.sock.listen(1)
self.modules = self.db.getModulePortsAndNames()
def startServer(self):
self.running = True
while self.running:
conn, addr = self.sock.accept()
self.handle(conn, addr)
conn.close()
self.sock.close()
def handle(self, conn, addr):
"""Handles a request"""
data = conn.recv(1024).strip('\n')
if data.startswith("CHECK TYPE "):
self.contestants = self.db.getClientIPs()
checktype = data.split(' ')[2]
IP = data.split(' ')[3]
if checktype == "NORMAL":
logging.info("[MANUALNORMAL] Starting check")
for module in self.modules:
logging.info("Checking module " + module['name'] + "on port " + str(module['port']))
result = checkIP(IP, module['port'])[0]
if not result['fine']:
logging.info("[MANUALNORMAL] Adding " + IP + " with port " + str(result['port']))
self.db.addSuspiciousContestant(IP, result['port'],'',module['name'])
logging.info("[MANUALNORMAL] Finished check")
elif checktype == "P2P":
logging.info("[MANUALP2P] Starting check")
for module in self.modules:
logging.info("[MANUALP2P] Checking module " + module['name'] + " on port " + str(module['port']))
self.targets = copy.copy(self.contestants)
self.targets.remove(IP)
p2p = P2PSanityCheck.PeerToPeerSanityChecker(IP,self.targets, module['port'])
p2p.checkIP()
results = p2p.getResults()
for client in results:
for result in client['results']:
#logging.info("%s reports port %s on %s: fine = %s" % (client['IP'], result['port'], IP, result['fine']))
if result['fine'] == 'False':
if result['port'] == "":
logging.info("[MANUALP2P] Adding " + client['IP'] + " for not running P2PRequestListener")
self.db.addSuspiciousContestant(client["IP"], "","", "PeerToPeerRequestListener")
else:
logging.info("[MANUALP2P] Adding " + IP + " with port " + str(result['port']) + "reported by " + client['IP'])
self.db.addSuspiciousContestant(IP, result['port'], client['IP'], module['name'])
logging.info("[MANUALP2P] Finished check.")
elif data == "STOPMANUAL":
logging.info("[MANUALSERVICE] Stopping Manual Sanity Check Service...")
self.running = False
def stopServer(self):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((self.host, self.port))
sock.send('STOPMANUAL')
sock.close()
def test_output_routes_outputs_single_route(self):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler.initialize_db()
handler.add_address(MapLocation(latitude=3, longitude=4, id=1))
handler.add_stop(MapLocation(latitude=9, longitude=10, id=1))
handler.add_route(address=1, stop=1, distance=50, time=100)
handler.output_routes(file_path='test_file.csv')
self.assertTrue(os.path.exists('test_file.csv'),
'test_file.csv file not found')
output = pd.read_csv('test_file.csv')
self.assertNotEqual(0, output.shape[0], "no output rows in output .csv")
self.assertEqual(3, output.ix[0, 'address_latitude'],
'incorrect address latitude output')
self.assertEqual(4, output.ix[0, 'address_longitude'],
'incorrect address longitude output')
self.assertEqual(9, output.ix[0, 'stop_latitude'],
'incorrect stop latitude output')
self.assertEqual(10, output.ix[0, 'stop_longitude'],
'incorrect stop longitude output')
self.assertEqual(50, output.ix[0, 'distance'],
'incorrect distance output')
self.assertEqual(100, output.ix[0, 'time'],
'incorrect time output')
class DAThread(QtCore.QObject):
onFolderUpdate = QtCore.pyqtSignal()
onDoubleUpdate = QtCore.pyqtSignal()
onProgressChange = QtCore.pyqtSignal(float)
def __init__(self, dbPath):
QtCore.QObject.__init__(self)
self.db = DatabaseHandler('DAThread', dbPath)
self.merger = Merger()
self.sha = QtCore.QCryptographicHash(QtCore.QCryptographicHash.Sha1)
return
def __del__(self):
return
@QtCore.pyqtSlot()
def closeEvent(self):
self.db = None
self.thread().exit()
return
@QtCore.pyqtSlot(QtCore.QString)
def onFolderAdded(self, path):
queries = [
('INSERT INTO folders (name, os, computerName) VALUES (:name, :os, :computerName)', {
':name': path,
':os': platform.system(),
':computerName': platform.node()
}
)
]
self.db.processQueries(queries)
self.onFolderUpdate.emit()
return
def sha1(self, path):
s = QtCore.QFile(path)
s.open(QtCore.QIODevice.ReadOnly)
self.sha.addData(s.readAll())
s.close()
res = self.sha.result().toHex()
self.sha.reset()
return res
def processFile(self, t):
absPath, path, fileName, size = t
query = 'INSERT INTO files (hash, absPath, name, path, size, searchTag) VALUES (:hash, :absPath, :name, :path, :size, :searchTag)'
dic = { ':hash': self.sha1(absPath), ':absPath': absPath, ':name': fileName, ':path': path, ':size': QtCore.QString(unicode(size)), ':searchTag': self.currentTag }
self.queries.append((query, dic))
return
def progressFun(self, percent):
if percent == 0:
return
elif percent != -1:
self.progress += percent
self.onProgressChange.emit(self.progress)
else:
self.onProgressChange.emit(-1)
return
def createSession(self):
timestamp = QtCore.QString(unicode(int(time.time())))
computerName = QtCore.QString(platform.node())
h = QtCore.QCryptographicHash(QtCore.QCryptographicHash.Sha1)
h.addData(computerName)
h.addData(timestamp)
currentTag = h.result().toHex()
query = 'INSERT INTO tags (searchTag, timestamp, computerName) VALUES (:searchTag, :timestamp, :computerName)'
dic = { ':searchTag': currentTag, ':timestamp': timestamp, ':computerName': computerName }
self.db.processQueries([(query, dic)])
self.queries = []
self.progress = 0.0
self.currentTag = currentTag
return
def saveProgress(self, d):
queries = self.queries
self.queries = []
query = 'REPLACE INTO lastSearch (searchTag, lastPercent, lastPath) VALUES (:searchTag, :lastPercent, :lastPath)'
dic = { ':searchTag': self.currentTag, ':lastPercent': self.progress, ':lastPath': d }
queries.append((query, dic))
self.db.processQueries(queries)
self.onDoubleUpdate.emit()
return
@QtCore.pyqtSlot(list, float)
def onSearchAsked(self, cl, percent):
self.createSession()
for d in cl:
walk(d, self.processFile, percent, self.progressFun)
self.saveProgress(d)
self.onDoubleUpdate.emit()
self.onProgressChange.emit(-1)
self.onDoubleUpdate.emit()
return
@QtCore.pyqtSlot(QtCore.QString)
def onFolderRemoved(self, path):
if path.isNull():
return
queries = [
('DELETE FROM folders where name = :name', {
':name': path
}
)
]
self.db.processQueries(queries)
self.onFolderUpdate.emit()
return
@QtCore.pyqtSlot(list, QtCore.QString)
def onMergerCalled(self, srcs, dst):
self.merger.merge(srcs, dst)
return
def __init__(self, dbPath):
QtCore.QObject.__init__(self)
self.db = DatabaseHandler('DAThread', dbPath)
self.merger = Merger()
self.sha = QtCore.QCryptographicHash(QtCore.QCryptographicHash.Sha1)
return
def test_get_address_without_route_generator_returns_generator(self):
handler = DatabaseHandler('unit_test_db.sqlite3')
self.assertIsInstance(handler.get_address_generator(),
types.GeneratorType)
class Interface:
"""
This class is high level interface, which combining data obtaining and analytics.
"""
def __init__(self, con_path):
self.__db_handler = DatabaseHandler(con_path)
def run_update(self, teams_to_update, threads, use_proxy, proxy_file, update_opponents):
for team_id in teams_to_update:
spider_configuration = {'threads': threads,
'use_proxy': use_proxy,
'proxy_file': proxy_file,
'ignore_id': self.__db_handler.get_matches_id(team_id)}
adapter = DotabuffAdapter(self.__db_handler, spider_configuration)
adapter.update_team(team_id)
print('Team Updated :: {}'.format(team_id))
if update_opponents:
adapter.update_opponents(team_id)
print('Team Opponents Updated :: {}'.format(team_id))
def run_backtesting(self, dotalounge_hist_file, dotalounge_names_file, window):
dotalounge_hist = pd.DataFrame.from_csv(dotalounge_hist_file, header=None)
dotalounge_names = pd.DataFrame.from_csv(dotalounge_names_file, header=0, index_col=None)
names = dotalounge_names['names'].tolist()
index = dotalounge_names['id'].tolist()
delta = datetime.timedelta(days=window)
BANKS = {}
unique_dates = list(set(dotalounge_hist.ix[:, 6].tolist()))
i = 0
for date_str in unique_dates:
date = datetime.datetime.strptime(date_str, '%Y-%m-%d')
games = dotalounge_hist.loc[(dotalounge_hist.ix[:, 6] == date_str)]
dates = [date - delta, date]
analytics = DotaBetsAnalytics(self.__db_handler.import_network(dates=dates))
methods = [analytics.marginal_winrate,
analytics.joint_winrate,
analytics.neighbors_winrate]
for game in games.iterrows():
teams_names = [game[1][1], game[1][2]]
if teams_names[0] in names and teams_names[1] in names:
try:
teams_id = [index[names.index(teams_names[0])], index[names.index(teams_names[1])]]
result = [0, 0]
result[game[1][3]] = 1
coeff = [game[1][4], game[1][5]]
probabilities = [result[0]]
for method_ind in range(len(methods)):
method_winrate = methods[method_ind](teams_id)
probabilities += [method_winrate[0], method_winrate[1]]
evs = [coeff[0] * method_winrate[0] - 1, coeff[1] * method_winrate[1] - 1]
rand_fav = randint(0, 1)
rand_out = abs(rand_fav - 1)
rand_won = result[rand_fav] * coeff[rand_fav] - 1
vs_book_fav = 0 if coeff[0] >= coeff[1] else 1
vs_book_out = abs(vs_book_fav - 1)
vs_book_won = result[vs_book_fav] * coeff[vs_book_fav] - 1
best_fav = 0 if result[0] == 1 else 1
best_won = result[best_fav] * coeff[best_fav] - 1
book_fav = 0 if coeff[0] <= coeff[1] else 1
book_out = abs(book_fav - 1)
book_won = result[book_fav] * coeff[book_fav] - 1
my_fav = 0 if evs[0] >= evs[1] else 1
my_out = abs(my_fav - 1)
if evs[my_fav] >= 0 and evs[my_out] < 0:
my_won = result[my_fav] * coeff[my_fav] - 1
my_ev = evs[my_fav]
else:
my_won = 0
my_ev = 0
if method_ind not in BANKS:
BANKS[method_ind] = DataFrame(columns=('rand_won',
'vs_book_won',
'book_won',
'my_won'))
result_row = [rand_won, vs_book_won, book_won, my_won]
BANKS[method_ind].loc[len(BANKS[method_ind])] = result_row
except:
pass
i += 1
print(i, len(unique_dates))
for bank in BANKS:
BANKS[bank].cumsum(0).plot()
plt.show()
def run_analytics(self, teams_list, coeffs_list, window):
delta = datetime.timedelta(days=window)
today = datetime.date.today()
dates = [today - delta, today]
analytics = DotaBetsAnalytics(self.__db_handler.import_network(dates=dates))
for i in range(len(teams_list)):
results = DataFrame(columns=['1', '2'])
teams = teams_list[i]
coeffs = coeffs_list[i]
try:
marginal_winrates = analytics.marginal_winrate(teams)
results.loc[len(results)] = [(marginal_winrates[0] * coeffs[0] - 1), (marginal_winrates[1] * coeffs[1] - 1)]
except:
results.loc[len(results)] = [-100, -100]
try:
joint_winrates = analytics.joint_winrate(teams)
results.loc[len(results)] = [(joint_winrates[0] * coeffs[0] - 1), (joint_winrates[1] * coeffs[1] - 1)]
except:
results.loc[len(results)] = [-100, -100]
try:
neighbors_winrates = analytics.neighbors_winrate(teams)
results.loc[len(results)] = [(neighbors_winrates[0] * coeffs[0] - 1), (neighbors_winrates[1] * coeffs[1] - 1)]
except:
results.loc[len(results)] = [-100, -100]
print('{} VS {}'.format(teams[0], teams[1]))
print(results)
def test_get_address_without_route_generator_yield_MapLocations(self):
handler = DatabaseHandler('unit_test_db.sqlite3')
handler.add_address(location=MapLocation(latitude=1, longitude=1))
address_generator = handler.get_address_generator()
self.assertIsInstance(address_generator.next(), MapLocation)
