class Receive_data( QtCore.QObject ): def __init__( self ): self.data = DataManager("data.db") QtCore.QObject.__init__(self) @QtCore.Slot('QString', 'QString') def execute(self,email,password): global client client = Client(email, password) response = client.login() if (response['result'] == 'missingParams' or response['result'] == 'invalidLogin'): Mbox('Budibox', 'Username or password incorrect. Please try again or register at http://www.budibox.com!') else: if (response['result'] == 'ok'): startup = client.notify_startup() if (not startup): print_message("Could not startup!") else: print_message("Notified!") self.data.addProperty("email", email); password = base64.b64encode(password) print password self.data.addProperty("password", str(password)); Mbox('Budibox', 'Login successful!') app.exit()
def add(self, task, projectName=None): date = Timings.now() if self.taskType(task) != "work": projectName = None attributes = self.processTask(date, task, projectName) DataManager.writeTask(date, task, projectName, firstToday=len(self.tasks) == 1) return attributes
def countTasks(self): """Count tasks statistics divided by projects""" self._data = DataManager.getByRange(self._fromDate, self._toDate) res = {} for date, task, projectName in self._data: if task == "__start__": self.timings.setPrevDate(None) spentSeconds = self.timings.count(date, Tasks.taskType(task)) if Tasks.taskType(task) != "work": continue if spentSeconds: if projectName not in res: res[projectName] = {} if task not in res[projectName]: res[projectName][task] = spentSeconds else: res[projectName][task] += spentSeconds self._countAttrib([v for k in res for v in res[k].values()]) if res: ret = {} for k in res.keys(): ret[k] = sorted(res[k].iteritems(), key=lambda item: item[1], reverse=True) return ret else: return {}
def _countObject(self, objType, targetAction): """Generic function for calculating projects data or slacking statistics""" self._data = DataManager.getByRange(self._fromDate, self._toDate) res = {} for date, task, projectName in self._data: if task == "__start__": self.timings.setPrevDate(None) objKey = projectName if objType == "project" else task spentSeconds = self.timings.count(date, Tasks.taskType(task)) if Tasks.taskType(task) != targetAction: self.timings.setPrevDate(date) continue if spentSeconds: if objKey not in res: res[objKey] = spentSeconds else: res[objKey] += spentSeconds self._countAttrib(res.values()) if res: return sorted(res.iteritems(), key=lambda item:item[1], reverse=True) else: return []
def __init__(self, top_right, bottom_left): self.event_instance = None self.data_manager = DataManager() self.top_right = top_right self.bottom_left = bottom_left self.result = self.run() self.data_manager.close_con()
def init_home_page(self, recommend_status="publish"): # Init DBus. self.system_bus = dbus.SystemBus() bus_object = self.system_bus.get_object(DSC_SERVICE_NAME, DSC_SERVICE_PATH) self.bus_interface = dbus.Interface(bus_object, DSC_SERVICE_NAME) # Say hello to backend. #self.bus_interface.say_hello(self.simulate) self.set_software_download_dir() self.inhibit_obj = InhibitObject() self.loginfo("Init data manager") # Init data manager. self.data_manager = DataManager(self.bus_interface, debug_flag) # Init packages status self.packages_status = {} # Init home page. self.home_page = HomePage(self.data_manager, recommend_status) # Init switch page. self.switch_page(self.home_page) self.in_update_list = False self.init_backend()
def __init__(self, dataset_path): """Init the engine with the dataset path """ print "Initializing DataManager" self.dataManager = DataManager(dataset_path) # url = 'http://commondatastorage.googleapis.com/books1000/' datastore_directory = "datastore/datasets/ml-latest-small" self.X_train, self.X_test, self.y_train, self.y_test, self.movies = self.dataManager.load_datasets(datastore_directory) #load model if one is saved self.model = self.dataManager.load_model("model.pkl")
class LoginBox: def __init__(self): self.data = DataManager("data.db") self.email = self.data.getPropertyValue("email") self.password = self.data.getPropertyValue("password") def start(self): global client if (len(self.email) > 0 and len(self.password) > 0): self.email = str(self.email[0]) self.password = str(base64.b64decode(self.password[0])) client = Client(self.email, self.password) response = client.login() if (response['result'] == 'ok'): startup = client.notify_startup() if (not startup): print_message("Could not startup!") else: print_message("Notified!") else: Mbox("Budibox", "Credentials undefined or incorrect. Please login again.") # Create the QML user interface. view = QDeclarativeView() view.setSource(QUrl('qml/main.qml')) view.setWindowTitle("Budibox") view.setWindowIcon(QIcon("qml/budibox.jpg")) context = view.rootContext() context.setContextProperty("send_data",Receive_data()) # Display the user interface and allow the user to interact with it. view.setGeometry(360, 360, 360, 360) view.setMaximumSize(360, 360) view.show() app.exec_()
def __init__(self): self.interval = 5.0 self.running = False self.configParser = ConfigParser.RawConfigParser() self.configParser.read('config/dob.cfg') self.data_manager = DataManager(configParser=self.configParser) self.drive_watcher = GDriveWatcher(configParser=self.configParser, data_manager=self.data_manager) self.slack_bot = SlackBot(configParser=self.configParser) self.witmanager = WitAIManager(configParser=self.configParser) self.last_drive_scan = None
def main(argv): manager = DataManager() manager.load() sess = tf.Session() model = VariationalAutoencoder(learning_rate=flags.learning_rate, beta=flags.beta) sess.run(tf.global_variables_initializer()) saver = load_checkpoints(sess) if flags.training: # Train train(sess, model, manager, saver) else: reconstruct_check_images = manager.get_random_images(10) # Image reconstruction check reconstruct_check(sess, model, reconstruct_check_images) # Disentangle check disentangle_check(sess, model, manager)
class MLEngine: """A machine learning engine """ def __init__(self, dataset_path): """Init the engine with the dataset path """ print "Initializing DataManager" self.dataManager = DataManager(dataset_path) # url = 'http://commondatastorage.googleapis.com/books1000/' datastore_directory = "datastore/datasets/ml-latest-small" self.X_train, self.X_test, self.y_train, self.y_test, self.movies = self.dataManager.load_datasets(datastore_directory) #load model if one is saved self.model = self.dataManager.load_model("model.pkl") def preprocess(self): print "Preprocessing data" def train(self): print "Training model" self.model.train(self.X_train, self.y_train, self.movies) # score on test data print self.model.score(self.X_test, self.y_test) def predict(self, X): prediction = self.model.predict(X) print prediction return prediction def validate(self): print "Validating..." def save_model(self, filename): print "Saving model to file..." self.dataManager.save_model(self.model, filename)
def __init__(self, batch_size, model_path, examples_path, max_image_width, train_test_ratio, restore): self.step = 0 self.__model_path = model_path self.__save_path = os.path.join(model_path, 'ckp') self.__restore = restore self.__training_name = str(int(time.time())) self.__session = tf.Session() # Building graph with self.__session.as_default(): (self.__inputs, self.__targets, self.__seq_len, self.__logits, self.__decoded, self.__optimizer, self.__acc, self.__cost, self.__max_char_count, self.__init) = self.crnn(max_image_width, batch_size) self.__init.run() with self.__session.as_default(): self.__saver = tf.train.Saver(tf.global_variables(), max_to_keep=10) # Loading last save if needed if self.__restore: print('Restoring') ckpt = tf.train.latest_checkpoint(self.__model_path) if ckpt: print('Checkpoint is valid') self.step = int(ckpt.split('-')[1]) self.__saver.restore(self.__session, ckpt) # Creating data_manager self.__data_manager = DataManager(batch_size, model_path, examples_path, max_image_width, train_test_ratio, self.__max_char_count)
class DataManagerTest(unittest.TestCase): def setUp(self): self.manager = DataManager() self.manager.load() def tearDown(self): del self.manager gc.collect() def test_load(self): # sample size should be 737280 self.assertEqual(self.manager.sample_size, 737280) def test_get_image(self): # get first image image0 = self.manager.get_image(shape=0, scale=0, orientation=0, x=0, y=0) self.assertTrue(image0.shape == (4096, )) # boundary check image1 = self.manager.get_image(shape=3 - 1, scale=6 - 1, orientation=40 - 1, x=32 - 1, y=32 - 1) self.assertTrue(image1.shape == (4096, )) def test_get_images(self): indices = [0, 1, 2] images = self.manager.get_images(indices) self.assertEqual(len(images), 3) # image shpe should be flatten. (4096) self.assertTrue(images[0].shape == (4096, )) def test_get_random_images(self): images = self.manager.get_random_images(3) self.assertEqual(len(images), 3) self.assertTrue(images[0].shape == (4096, ))
def main(): flight_search = FlightSearch() data_manager = DataManager() notification_manager = NotificationManager() data = data_manager.get_data()["prices"] # Get IATA Codes for cities in spreadsheet. if data[0]["iataCode"] == "": city_ids = [(elem["id"], elem["city"]) for elem in data] for elem in city_ids: iata = flight_search.get_iata_code_by_city(elem[1]) body = { "price": { "iataCode": iata, } } data_manager.put_data(body, elem[0]) # Get Cheap flights from LON (London) to destinations in spread sheet. for destination in data: flight = flight_search.search_a_flight(destination["iataCode"]) if flight is None: continue # Notify if cheap flight is found. if destination["lowestPrice"] > flight.price: message = "Low price alert!\n" \ f"Only {flight.price} to fly from {flight.origin_city}-{flight.origin_airport}" \ f" to {flight.destination_city}-{flight.destination_airport}" \ f" from {flight.out_date}" \ f" to {flight.return_date}." if flight.stop_overs > 0: message += f"\nFlight has {flight.stop_overs} stop over, via {flight.via_city}." # Notify me on phone notification_manager.send_message(message) #notify customers via mail customers = data_manager.get_users()["users"] notification_manager.send_emails(customers[0]["email"], flight)
def get_features(base_path='../features/'): n = 115615 # n = 500 labels = DataManager.get_labels_2_classes().iloc[0:n, 1] # features = pd.read_csv(base_path + 'labels_to_test.csv').iloc[0:n, 2:3] embeddings = DataManager.get_embeddings().iloc[0:n, :] n_grams = DataManager.get_ngrams().iloc[0:n] scores = DataManager.get_scores().iloc[0:n, 2:] pos_tags = DataManager.get_pos_ngrams().iloc[0:n] linguistic = DataManager.get_linguistic().iloc[0:n] print('Merging features...') features = embeddings.merge(linguistic, on='0') features = n_grams.merge(features, left_on='rev_id', right_on='0').drop('0', axis=1) features = features.merge(scores, on='rev_id') features = features.merge(pos_tags, on='rev_id') return features.values, labels
class DataManagerTest(unittest.TestCase): def setUp(self): self.manager = DataManager() self.manager.load() def tearDown(self): del self.manager gc.collect() def test_load(self): # sample size should be 737280 self.assertEquals(self.manager.sample_size, 737280) def test_get_image(self): # get first image image0 = self.manager.get_image(shape=0, scale=0, orientation=0, x=0, y=0) self.assertTrue(image0.shape == (4096,)) # boundary check image1 = self.manager.get_image(shape=3-1, scale=6-1, orientation=40-1, x=32-1, y=32-1) self.assertTrue(image1.shape == (4096,)) def test_get_images(self): indices = [0,1,2] images = self.manager.get_images(indices) self.assertEquals(len(images), 3) # image shpe should be flatten. (4096) self.assertTrue(images[0].shape == (4096,)) def test_get_random_images(self): images = self.manager.get_random_images(3) self.assertEquals(len(images), 3) self.assertTrue(images[0].shape == (4096,))
class TeamBuilder: def __init__(self): self.data_manager = DataManager() # Solution as a knapsack problem with restrictions using LP. def team_builder(self, budget): defender_pos = ['RCB', 'CB', 'LB', 'RB', 'RWB', 'LWB'] middle_pos = [ 'RCM', 'LCM', 'LDM', 'CDM', 'LCB', 'RM', 'LM', 'RDM', 'CM' ] forward_pos = [ 'RF', 'ST', 'LW', 'LF', 'RS', 'CAM', 'LS', 'LAM', 'RW', 'RAM', 'CF' ] data = self.data_manager.get_data_frame_from_db() player = [str(i) for i in range(data.shape[0])] point = {str(i): data['Overall'][i] for i in range(data.shape[0])} value = {str(i): data['Value'][i] for i in range(data.shape[0])} gk = { str(i): 1 if data['Position'][i] == 'GK' else 0 for i in range(data.shape[0]) } fullback = { str(i): 1 if data['Position'][i] in defender_pos else 0 for i in range(data.shape[0]) } halfback = { str(i): 1 if data['Position'][i] in middle_pos else 0 for i in range(data.shape[0]) } forward = { str(i): 1 if data['Position'][i] in forward_pos else 0 for i in range(data.shape[0]) } xi = {str(i): 1 for i in range(data.shape[0])} prob = LpProblem("Knapsack_Football", LpMaximize) player_vars = LpVariable.dicts("Players", player, 0, 1, LpBinary) # objective function prob += lpSum([point[i] * player_vars[i] for i in player]), "Total_Cost" # constraint prob += lpSum([player_vars[i] for i in player]) == 11, "Total_11_Players" prob += lpSum([value[i] * player_vars[i] for i in player]) <= budget, "Total_Cost" prob += lpSum([gk[i] * player_vars[i] for i in player]) == 1, "1_GK" prob += lpSum([fullback[i] * player_vars[i] for i in player]) == 2, "2_DEF" prob += lpSum([halfback[i] * player_vars[i] for i in player]) == 3, "3_MID" prob += lpSum([forward[i] * player_vars[i] for i in player]) == 5, "5_STR" # solve status = prob.solve() player_index_list = [] for v in prob.variables(): if v.varValue > 0: index = str(v.name).replace("Players_", '') player_index_list.append(int(index)) result = data.iloc[player_index_list] return result
def build_data_manager(frame_folder, feature_folder, caption_file, key_frame_info_folder, save_path): data_manager = DataManager() data_manager.load_frame_path_info(frame_folder) data_manager.load_captions(caption_file) data_manager.load_key_frame_information(key_frame_info_folder) data_manager.load_features(feature_folder) data_manager.save(save_path)
class OrderHandler: def __init__(self, server=requests): self.server = server self.data = DataManager(self.server) """ Interface Methods """ def start_interface(self) -> int: while True: print("\nCommand list:") print("submit_order") print("update_order") print("cancel_order") print("print_menu") print("quit") command = input("\nEnter your command: ") if command == "quit": break elif command == "submit_order": self.submit_order() elif command == "update_order": self.update_order() elif command == "cancel_order": self.cancel_order() elif command == "print_menu": self.print_menu() else: print("Error: Invalid command") def submit_order(self) -> None: pizza_size = "" pizza_type = "" toppings = [] drink = "" delivery_type = "" while True: pizza_size = input("Select a size: ") if self.check_pizza_size(pizza_size): break print("Error: Invalid size") while True: pizza_type = input("Select a pizza type (or enter 'None'): ") if self.check_pizza_type(pizza_type): break print("Error: Invalid type") while True: topping = input("Add a pizza topping (or enter 'None'): ") if topping.lower() == 'none': break elif not self.check_topping(topping): print("Error: Invalid topping") else: toppings.append(topping) while True: drink = input("Select a drink: ") if self.check_drink(drink): break print("Error: Invalid drink") while True: delivery_type = input("Select a delivery type: ") if self.check_delivery_type(delivery_type): break print("Error: Invalid delivery type") address = input("Enter your address (enter anything if pick-up): ") order_no = self.submit_order_to_server(pizza_size, pizza_type, toppings, drink, delivery_type, address) print("Your order number is: " + str(order_no)) print("Lottory Chance, Pull 777 to win for free food") lotto = input("Pull? (y/n): ") if lotto == "y" or lotto == "Y": pull_num = self.lottery() print("You pulled: " + str(pull_num)) if pull_num == 777: print("You won!!!") print("Your total is 0 dollars.") else: print("Your total is {} dollars.".format( self.calculate_total(self.get_order_by_no(order_no)))) else: print("Your total is {} dollars.".format( self.calculate_total(self.get_order_by_no(order_no)))) def update_order(self) -> None: order_no = input("Enter order number: ") print("Select an item to update") item = input("size/type/topping/drink/delivery/address): ") add_topping = False if item.lower() == "topping": ans = input("Add or remove topping? (a/r): ") if ans == "a": add_topping = True new_value = input("Enter the item to update: ") n = self.update_order_in_server(order_no, item, new_value, add_topping) if n == 0: print("Updated successfully") elif n == 1: print("Nothing updated") else: print("Failed to update") def cancel_order(self) -> None: order_no = input("Enter order number: ") if self.cancel_order_in_server(order_no) == 0: print("Order " + order_no + " cancelled successfully") else: print("Error: Invalid order number") def print_menu(self) -> None: print("Pizza Types: ") types = self.data.get_pizza_manager().get_pizza_types() for i in range(len(types)): print( str(i + 1) + ". " + types[i]["Type"] + " $" + str(types[i]["Price"])) print() print("Pizza Toppings: ") toppings = self.data.get_pizza_manager().get_pizza_toppings() for i in range(len(toppings)): print( str(i + 1) + ". " + toppings[i]["Topping"] + " $" + str(toppings[i]["Price"])) print() print("Pizza Sizes: ") sizes = self.data.get_pizza_manager().get_pizza_sizes() for i in range(len(sizes)): print( str(i + 1) + ". " + sizes[i]["Size"] + " $" + str(sizes[i]["Price"])) print() print("Pizza Drinks: ") drinks = self.data.get_pizza_manager().get_drinks() for i in range(len(drinks)): print( str(i + 1) + ". " + drinks[i]["Drink"] + " $" + str(drinks[i]["Price"])) print() print("Delivery Types: ") delivery_types = self.data.get_delivery_manager().get_delivery_types() for delivery_type in delivery_types: print(delivery_type) print() """ Processing Methods """ def check_pizza_size(self, pizza_size: str) -> bool: size_dict = self.data.get_pizza_manager().get_pizza_sizes() for obj in size_dict: if obj["Size"].lower() == pizza_size.lower(): return True return False def check_pizza_type(self, pizza_type: str) -> bool: if pizza_type.lower() == "none": return True type_dict = self.data.get_pizza_manager().get_pizza_types() for obj in type_dict: if obj["Type"].lower() == pizza_type.lower(): return True return False def check_topping(self, topping: str) -> bool: topping_dict = self.data.get_pizza_manager().get_pizza_toppings() for obj in topping_dict: if obj["Topping"].lower() == topping.lower(): return True return False def check_drink(self, drink: str) -> bool: drink_dict = self.data.get_pizza_manager().get_drinks() for obj in drink_dict: if obj["Drink"].lower() == drink.lower(): return True return False def check_delivery_type(self, delivery_type: str) -> bool: type_list = self.data.get_delivery_manager().get_delivery_types() for t in type_list: if t == delivery_type.lower(): return True return False def get_order_by_no(self, order_no: int) -> Order: raw_order = self.data.get_order_manager().get_order(order_no) if raw_order == 404: return None order = Order.from_json(json.loads(raw_order)) return order def get_delivery_by_type_no(self, delivery_type: str, delivery_no: int) -> Delivery: raw_delivery = self.data.get_delivery_manager().get_each_delivery( delivery_type, delivery_no) if raw_delivery == 404: return None delivery = Delivery.from_json(raw_delivery) return delivery def submit_order_to_server(self, pizza_size: str, pizza_type: str, toppings: [str], drink: str, delivery_type: str, address: str) -> int: if not pizza_type.lower() == "none": predefined = self.data.get_pizza_manager().get_predefined() predefined_toppings = [] for d in predefined: if pizza_type in d: predefined_toppings = d[pizza_type] for t in predefined_toppings: if t not in toppings: toppings.append(t) order = OrderCreator.get_instance().create_order( pizza_size, pizza_type, toppings, drink, delivery_type) self.data.get_order_manager().add_new_order(order.get_order_no(), order) delivery = DeliveryCreator.get_instance().create_delivery( address, order, order.get_order_no()) self.data.get_delivery_manager().add_new_delivery( order.get_delivery(), delivery, order.get_delivery_no()) return order.get_order_no() def update_order_in_server(self, order_no: int, item: str, new_value: str, add_topping: bool) -> int: """ Return 0 if updated successfully. Otherwise return -1. """ order = self.get_order_by_no(order_no) delivery = self.get_delivery_by_type_no(order.get_delivery(), order_no) if item == "size": if not self.check_pizza_size(new_value): print("Error: Invalid size") return -1 else: order.set_pizza_size(new_value) self.data.get_order_manager().update_order( order.get_order_no(), order) delivery.set_order(order) self.data.get_delivery_manager().update_delivery( order.get_delivery(), delivery, order.get_delivery_no()) return 0 elif item == "type": if not self.check_pizza_type(new_value): print("Error: Invalid type") return -1 else: order.set_pizza_type(new_value) self.data.get_order_manager().update_order( order.get_order_no(), order) delivery.set_order(order) self.data.get_delivery_manager().update_delivery( order.get_delivery(), delivery, order.get_delivery_no()) return 0 elif item == "topping": if not self.check_topping(new_value): print("Error: Invalid topping") return -1 else: if add_topping and new_value not in order.get_toppings(): order.add_topping(new_value) elif not add_topping and new_value in order.get_toppings(): order.remove_topping(new_value) else: return 1 self.data.get_order_manager().update_order( order.get_order_no(), order) delivery.set_order(order) self.data.get_delivery_manager().update_delivery( order.get_delivery(), delivery, order.get_delivery_no()) return 0 elif item == "drink": if not self.check_drink(new_value): print("Error: Invalid drink") return -1 else: order.set_drink(new_value) self.data.get_order_manager().update_order( order.get_order_no(), order) delivery.set_order(order) self.data.get_delivery_manager().update_delivery( order.get_delivery(), delivery, order.get_delivery_no()) return 0 elif item == "delivery": if not self.check_delivery_type(new_value): print("Error: Invalid delivery type") return -1 else: order.set_delivery(new_value) self.data.get_order_manager().update_order( order.get_order_no(), order) delivery.set_order(order) self.data.get_delivery_manager().update_delivery( new_value, delivery, delivery.get_delivery_no()) return 0 elif item == "address": delivery.set_address(new_value) self.data.get_delivery_manager().update_delivery( order.get_delivery(), delivery, delivery.get_delivery_no()) return 0 else: print("Error: Invalid item to update") return -1 def cancel_order_in_server(self, order_no) -> int: """ Return 0 if order_no is valid and the order is deleted successfully. Otherwisw return -1. """ order = self.get_order_by_no(order_no) if order is None: return -1 status1 = self.data.get_delivery_manager().delete_delivery( order.get_delivery(), order.get_delivery_no()) status2 = self.data.get_order_manager().delete_order(order_no) if status1 == 404 or status2 == 404: return -1 return 0 def check_json_type(self) -> bool: """ Return True if server is of type requests Return False if server is of type flask.request """ return isinstance(self.server, requests) def calculate_total(self, order: Order) -> float: if order is None: return 0 size_price = 0 sizes = self.data.get_pizza_manager().get_pizza_sizes() for size in sizes: if size["Size"].lower() == order.get_pizza_size().lower(): size_price = size["Price"] break type_price = 0 types = self.data.get_pizza_manager().get_pizza_types() for type in types: if type["Type"].lower() == order.get_pizza_type().lower(): type_price = type["Price"] break topping_price = 0 toppings = self.data.get_pizza_manager().get_pizza_toppings() for order_topping in order.get_toppings(): for topping in toppings: if topping["Topping"].lower() == order_topping.lower(): topping_price += topping["Price"] break drink_price = 0 drinks = self.data.get_pizza_manager().get_drinks() for drink in drinks: if drink["Drink"].lower() == order.get_drink().lower(): drink_price = drink["Price"] break return size_price + type_price + topping_price + drink_price def lottery(self): n = self.data.get_order_manager().get_seed_for_lotto() random.seed(n) n = int(random.random() * 1000) self.data.get_order_manager().set_seed_for_lotto(n) return n
def transform(self, X, y=None): return self.transformer.transform(X) if __name__ == "__main__": # We can use this to run this file as a script and test the Preprocessor if True: # Use the default input and output directories if no arguments are provided input_dir = "C:\\Users\\isabe\\Downloads\\monet-master\\starting_kit\\c1_input_data" output_dir = "./fichiers_preprocesses" else: input_dir = argv[1] output_dir = argv[2] basename = 'perso' D = DataManager(basename, input_dir) # Load data print("*** Original data ***") print(D) Prepro = Preprocessor() # Preprocess on the data and load it back into D D.data['X_train'] = Prepro.fit_transform(D.data['X_train'], D.data['Y_train']) D.data['X_valid'] = Prepro.transform(D.data['X_valid']) D.data['X_test'] = Prepro.transform(D.data['X_test']) #D.feat_name = np.array(['PC1', 'PC2']) # D.feat_type = np.array(['Numeric', 'Numeric']) # Here show something that proves that the preprocessing worked fine print("*** Transformed data ***")
from data_manager import DataManager from analysis_models.airport_model import AirportModel from analysis_models.bag_model import BagModel import matplotlib.pyplot as plt import pandas as pd if __name__ == '__main__': data_folder = '../data/' output_folder = '../output/' airport_metadata_filename = 'DimFlyplassProccesed.csv' amount_files = 62 data_file_1 = 0 dm = DataManager(data_folder, output_folder, amount_files, airport_metadata_filename) # Import data dm.read_multiple_data_files() bags = dm.bag_messages bags['sourceTimestamp'] = pd.to_datetime(bags['sourceTimestamp']) bags.sort_values(by='sourceTimestamp') print(len(bags)) filtered = bags[bags['bagEventCode'].map( lambda x: str(x) == 'BagTagGenerated')] print(len(filtered)) bagsByDay = filtered['sourceTimestamp'].groupby( filtered['sourceTimestamp'].dt.floor('d')).size().reset_index( name='count') print(len(bagsByDay)) print(bagsByDay)
from data_manager import DataManager data_manager = DataManager() print("Welcome to Lui's Flight Club.\nWe're dedicated to finding the best flight deals for you and will notify you " "through email.") continue_verification = True first_name = input("What is your first name?\n").capitalize() last_name = input("What is your last name?\n").capitalize() user_email = input("What is your email?\n") while continue_verification: email_confirmation = input("Type your email again, please.\n") if email_confirmation == user_email: print("Alright! You're in the club.") data_manager.add_new_user( first_name, last_name, user_email) continue_verification = False else: print("Emails don't match.") continue
from data_manager import DataManager from flight_search import FlightSearch from flight_data import FlightData from notification_manager import NotificationManager data_manager = DataManager() sheet_data = data_manager.get_data() if sheet_data[0]['iataCode'] == "": flight_search = FlightSearch() for city in sheet_data: city["iataCode"] = flight_search.get_city_code(city["city"]) print(f"sheet_data:\n {sheet_data}") data_manager.destination_data = sheet_data data_manager.input_data() flight_data = FlightData() for city in sheet_data: data = flight_data.flight_info(city["iataCode"]) if data == "unavailable": print(f"{city['city']}: not available, even with stopover") continue else: price = data["price"] print(f"{city['city']}: £{price}") if price is not None: if int(price) < city["lowestPrice"]: routes = data['route'] depart = routes[0]['local_arrival'].split("T")[0]
path.append ("../ingestion_program") # Contains libraries you will need from data_manager import DataManager # such as DataManager import prepro if __name__=="__main__": # We can use this to run this file as a script and test the Preprocessor if len(argv)==1: # Use the default input and output directories if no arguments are provided input_dir = "../public_data" output_dir = "../results" # Create this directory if it does not exist else: input_dir = argv[1] output_dir = argv[2]; basename = 'air' D = DataManager(basename, input_dir) # Load data print("*** Original data ***") # On effectue le test de notre preprocessing de façon très simple, en deux affichages : # Tout d'abord on vérifie que les données brutes ont 14 features contenant des valeurs entières. print("Nombre de features : {0}.".format(D.data['X_train'].shape[1])) print(D.data['X_train'][0,:]) Prepro = prepro.Preprocessor() # Preprocess on the data and load it back into D D.data['X_train'] = Prepro.fit_transform(D.data['X_train'], D.data['Y_train']) D.data['X_valid'] = Prepro.fit_transform(D.data['X_valid']) D.data['X_test'] = Prepro.fit_transform(D.data['X_test']) # Here show something that proves that the preprocessing worked fine print("*** Transformed data ***")
def get_id_from_name( commodityName ): from data_manager import DataManager DataManager.init() commodityName = commodityName.lower() return DataManager.nameToId[ commodityName ] if commodityName in DataManager.nameToId else None
def __init__(self): # Make sure all the directories exist if not tf.gfile.Exists(TFLOG_PATH): tf.gfile.MakeDirs(TFLOG_PATH) if not tf.gfile.Exists(EXPERIENCE_PATH): tf.gfile.MakeDirs(EXPERIENCE_PATH) if not tf.gfile.Exists(NET_SAVE_PATH): tf.gfile.MakeDirs(NET_SAVE_PATH) # Initialize our session self.session = tf.Session() self.graph = self.session.graph with self.graph.as_default(): # View the state batches self.visualize_input = VISUALIZE_BUFFER if self.visualize_input: self.viewer = CostmapVisualizer() # Hardcode input size and action size self.height = 86 self.width = self.height self.depth = 4 self.action_dim = 2 # Initialize the current action and the old action and old state for setting experiences self.old_state = np.zeros((self.width, self.height, self.depth), dtype='int8') self.old_action = np.ones(2, dtype='float') self.network_action = np.zeros(2, dtype='float') self.noise_action = np.zeros(2, dtype='float') self.action = np.zeros(2, dtype='float') # Initialize the grad inverter object to keep the action bounds self.grad_inv = GradInverter(A0_BOUNDS, A1_BOUNDS, self.session) # Make sure the directory for the data files exists if not tf.gfile.Exists(DATA_PATH): tf.gfile.MakeDirs(DATA_PATH) # Initialize summary writers to plot variables during training self.summary_op = tf.merge_all_summaries() self.summary_writer = tf.train.SummaryWriter(TFLOG_PATH) # Initialize actor and critic networks self.actor_network = ActorNetwork(self.height, self.action_dim, self.depth, self.session, self.summary_writer) self.critic_network = CriticNetwork(self.height, self.action_dim, self.depth, self.session, self.summary_writer) # Initialize the saver to save the network params self.saver = tf.train.Saver() # initialize the experience data manger self.data_manager = DataManager(BATCH_SIZE, EXPERIENCE_PATH, self.session) # Uncomment if collecting a buffer for the autoencoder # self.buffer = deque() # Should we load the pre-trained params? # If so: Load the full pre-trained net # Else: Initialize all variables the overwrite the conv layers with the pretrained filters if PRE_TRAINED_NETS: self.saver.restore(self.session, NET_LOAD_PATH) else: self.session.run(tf.initialize_all_variables()) tf.train.start_queue_runners(sess=self.session) time.sleep(1) # Initialize a random process the Ornstein-Uhlenbeck process for action exploration self.exploration_noise = OUNoise(self.action_dim, MU, THETA, SIGMA) self.noise_flag = True # Initialize time step self.training_step = 0 # Flag: don't learn the first experience self.first_experience = True # After the graph has been filled add it to the summary writer self.summary_writer.add_graph(self.graph)
class DDPG: def __init__(self): # Make sure all the directories exist if not tf.gfile.Exists(TFLOG_PATH): tf.gfile.MakeDirs(TFLOG_PATH) if not tf.gfile.Exists(EXPERIENCE_PATH): tf.gfile.MakeDirs(EXPERIENCE_PATH) if not tf.gfile.Exists(NET_SAVE_PATH): tf.gfile.MakeDirs(NET_SAVE_PATH) # Initialize our session self.session = tf.Session() self.graph = self.session.graph with self.graph.as_default(): # View the state batches self.visualize_input = VISUALIZE_BUFFER if self.visualize_input: self.viewer = CostmapVisualizer() # Hardcode input size and action size self.height = 86 self.width = self.height self.depth = 4 self.action_dim = 2 # Initialize the current action and the old action and old state for setting experiences self.old_state = np.zeros((self.width, self.height, self.depth), dtype='int8') self.old_action = np.ones(2, dtype='float') self.network_action = np.zeros(2, dtype='float') self.noise_action = np.zeros(2, dtype='float') self.action = np.zeros(2, dtype='float') # Initialize the grad inverter object to keep the action bounds self.grad_inv = GradInverter(A0_BOUNDS, A1_BOUNDS, self.session) # Make sure the directory for the data files exists if not tf.gfile.Exists(DATA_PATH): tf.gfile.MakeDirs(DATA_PATH) # Initialize summary writers to plot variables during training self.summary_op = tf.merge_all_summaries() self.summary_writer = tf.train.SummaryWriter(TFLOG_PATH) # Initialize actor and critic networks self.actor_network = ActorNetwork(self.height, self.action_dim, self.depth, self.session, self.summary_writer) self.critic_network = CriticNetwork(self.height, self.action_dim, self.depth, self.session, self.summary_writer) # Initialize the saver to save the network params self.saver = tf.train.Saver() # initialize the experience data manger self.data_manager = DataManager(BATCH_SIZE, EXPERIENCE_PATH, self.session) # Uncomment if collecting a buffer for the autoencoder # self.buffer = deque() # Should we load the pre-trained params? # If so: Load the full pre-trained net # Else: Initialize all variables the overwrite the conv layers with the pretrained filters if PRE_TRAINED_NETS: self.saver.restore(self.session, NET_LOAD_PATH) else: self.session.run(tf.initialize_all_variables()) tf.train.start_queue_runners(sess=self.session) time.sleep(1) # Initialize a random process the Ornstein-Uhlenbeck process for action exploration self.exploration_noise = OUNoise(self.action_dim, MU, THETA, SIGMA) self.noise_flag = True # Initialize time step self.training_step = 0 # Flag: don't learn the first experience self.first_experience = True # After the graph has been filled add it to the summary writer self.summary_writer.add_graph(self.graph) def train(self): # Check if the buffer is big enough to start training if self.data_manager.enough_data(): # get the next random batch from the data manger state_batch, \ action_batch, \ reward_batch, \ next_state_batch, \ is_episode_finished_batch = self.data_manager.get_next_batch() state_batch = np.divide(state_batch, 100.0) next_state_batch = np.divide(next_state_batch, 100.0) # Are we visualizing the first state batch for debugging? # If so: We have to scale up the values for grey scale before plotting if self.visualize_input: state_batch_np = np.asarray(state_batch) state_batch_np = np.multiply(state_batch_np, -100.0) state_batch_np = np.add(state_batch_np, 100.0) self.viewer.set_data(state_batch_np) self.viewer.run() self.visualize_input = False # Calculate y for the td_error of the critic y_batch = [] next_action_batch = self.actor_network.target_evaluate(next_state_batch) q_value_batch = self.critic_network.target_evaluate(next_state_batch, next_action_batch) for i in range(0, BATCH_SIZE): if is_episode_finished_batch[i]: y_batch.append([reward_batch[i]]) else: y_batch.append(reward_batch[i] + GAMMA * q_value_batch[i]) # Now that we have the y batch lets train the critic self.critic_network.train(y_batch, state_batch, action_batch) # Get the action batch so we can calculate the action gradient with it # Then get the action gradient batch and adapt the gradient with the gradient inverting method action_batch_for_gradients = self.actor_network.evaluate(state_batch) q_gradient_batch = self.critic_network.get_action_gradient(state_batch, action_batch_for_gradients) q_gradient_batch = self.grad_inv.invert(q_gradient_batch, action_batch_for_gradients) # Now we can train the actor self.actor_network.train(q_gradient_batch, state_batch) # Save model if necessary if self.training_step > 0 and self.training_step % SAVE_STEP == 0: self.saver.save(self.session, NET_SAVE_PATH, global_step=self.training_step) # Update time step self.training_step += 1 self.data_manager.check_for_enqueue() def get_action(self, state): # normalize the state state = state.astype(float) state = np.divide(state, 100.0) # Get the action self.action = self.actor_network.get_action(state) # Are we using noise? if self.noise_flag: # scale noise down to 0 at training step 3000000 if self.training_step < MAX_NOISE_STEP: self.action += (MAX_NOISE_STEP - self.training_step) / MAX_NOISE_STEP * self.exploration_noise.noise() # if action value lies outside of action bounds, rescale the action vector if self.action[0] < A0_BOUNDS[0] or self.action[0] > A0_BOUNDS[1]: self.action *= np.fabs(A0_BOUNDS[0]/self.action[0]) if self.action[1] < A0_BOUNDS[0] or self.action[1] > A0_BOUNDS[1]: self.action *= np.fabs(A1_BOUNDS[0]/self.action[1]) # Life q value output for this action and state self.print_q_value(state, self.action) return self.action def set_experience(self, state, reward, is_episode_finished): # Make sure we're saving a new old_state for the first experience of every episode if self.first_experience: self.first_experience = False else: self.data_manager.store_experience_to_file(self.old_state, self.old_action, reward, state, is_episode_finished) # Uncomment if collecting data for the auto_encoder # experience = (self.old_state, self.old_action, reward, state, is_episode_finished) # self.buffer.append(experience) if is_episode_finished: self.first_experience = True self.exploration_noise.reset() # Safe old state and old action for next experience self.old_state = state self.old_action = self.action def print_q_value(self, state, action): string = "-" q_value = self.critic_network.evaluate([state], [action]) stroke_pos = 30 * q_value[0][0] + 30 if stroke_pos < 0: stroke_pos = 0 elif stroke_pos > 60: stroke_pos = 60 print '[' + stroke_pos * string + '|' + (60-stroke_pos) * string + ']', "Q: ", q_value[0][0], \ "\tt: ", self.training_step
def __init__(self): self.data = DataManager("data.db") self.email = self.data.getPropertyValue("email") self.password = self.data.getPropertyValue("password")
if existing_entity is None or existing_entity.lastSlackUpdate is None or \ existing_entity.lastSlackUpdate <= existing_entity.modifiedDate: entity.lastSlackUpdate = datetime.datetime.now() new_and_updated_entities.append(entity) self.data_manager.put_entity(entity) print "adding entity {}".format(entity.id) #self.get_file_content(entity.id) return new_and_updated_entities def process_updated_and_new_entities(self, new_and_updated_entities): pass if __name__ == '__main__': from data_manager import DataManager configParser = ConfigParser.RawConfigParser() configParser.read('config/dob.cfg') data_manager = DataManager(configParser=configParser) data_manager.connect() watcher = GDriveWatcher(configParser=configParser, data_manager=data_manager) watcher.connect() watcher.update()
def __init__( self ): self.data = DataManager("data.db") QtCore.QObject.__init__(self)
from data_manager import DataManager from flight_search import FlightSearch from datetime import datetime, timedelta from notification_manager import NotificationManager ORIGIN_CITY_CODE = "NYC" # Step 3 data_manager = DataManager() sheet_data = data_manager.get_data_destination() customer_data = data_manager.get_customer_emails() flight_search = FlightSearch() notificator = NotificationManager() # If the city in the spreadsheet doesn't have a IATA Code this part will get it and update the spreadsheet for row in range(len(sheet_data)): if sheet_data[row]['IATA Code'] == '': sheet_data[row]['IATA Code'] = flight_search.get_city_code( sheet_data[row]['City']) data_manager.data_destination = sheet_data data_manager.update_destination_code() # Obtain dates tomorrow = datetime.now() + timedelta(days=1) six_months_from_now = tomorrow + timedelta(days=(6 * 30)) found_flights = True for destination in sheet_data: flight = flight_search.search_for_flight( ORIGIN_CITY_CODE, destination['IATA Code'], from_date=tomorrow,
import requests from flight_search import FlightSearch from notification_manager import NotificationManager from datetime import datetime from data_manager import DataManager from flight_data import FlightData #This file will need to use the DataManager,FlightSearch, FlightData, NotificationManager classes to achieve the program requirements. SHEET_API_KEY = "3d0f05f887e0580677d4c0878ad2d460" data_manager = DataManager(SHEET_API_KEY) sheet_data = data_manager.get_data()['prices'] flight_data = FlightData() # print(flight_data.api_call("PAR")) notification = NotificationManager() for data in sheet_data: price = flight_data.api_call(data['iataCode']) for lowest in price: if data['lowestPrice'] > lowest['price']: notification.send_message( f"Low price alert! Only ${lowest['price']} to fly from " f"{lowest['cityFrom']}-{lowest['flyFrom']}" f" to {lowest['cityTo']}-{lowest['flyTo']}" f" from {datetime.now().date()} to " f"{lowest['utc_departure'].split('T')[0]}") break # for data in sheet_data: # if data['iataCode'] == "":
def add_polygon_with_circles_and_label(self, path, linecolor=None, linewidth=None, vertex_color=None, vertex_radius=None, label='unknown', label_pos=None, section=None, index=None, type=None, edits=[], side=None, side_manually_assigned=None, contour_id=None, position=None, category='contour', **kwargs): """ Additional keyword arguments are used to set polygon properties. - set_name (str): specifies the set this polygon belongs to (handdrawn or aligned_atlas) Args: type (str): One of the following, - derived_from_atlas - confirmed (hand-drawn or confirming interpolated polygons) - interpolated """ closed = polygon_is_closed(path=path) vertices = vertices_from_polygon(path=path, closed=closed) if len(vertices) == 2: raise Exception("Polygon has only two vertices. Skip adding this polygon.") if index is not None: assert isinstance(index, int), "add_polygon_with_circles_and_label(): Argument `index` must be integer." polygon = self.add_polygon_with_circles(path, linecolor=linecolor, linewidth=linewidth, vertex_color=vertex_color, vertex_radius=vertex_radius, section=section, index=index) polygon.signal_emitter.property_changed.connect(self.polygon_property_changed) # Compute the polygon's coordinate in the depth dimension. if hasattr(self.data_feeder, 'sections'): if section is None: section = self.data_feeder.sections[index] position = DataManager.convert_section_to_z(sec=section, resolution=self.data_feeder.resolution, mid=True, stack=self.gui.stack) # z0, z1 = self.convert_section_to_z(sec=section, downsample=self.data_feeder.downsample, mid=True) # position = (z0 + z1) / 2 else: position = index polygon.set_properties('position', position) polygon.set_properties('label', label) if label_pos is not None: polygon.set_properties('label_pos', label_pos) polygon.set_properties('type', type) polygon.set_properties('side', side) polygon.set_properties('side_manually_assigned', side_manually_assigned) polygon.set_properties('contour_id', contour_id) # Could be None - will be generated new in convert_drawings_to_entries() polygon.set_properties('orientation', self.data_feeder.orientation) if edits is None or len(edits) == 0: polygon.set_properties('edits', [{'username': self.gui.get_username(), 'timestamp': datetime.now().strftime("%m%d%Y%H%M%S")}]) else: polygon.set_properties('edits', edits) polygon.set_properties('class', category) if hasattr(self.data_feeder, 'sections'): polygon.set_properties('section', section) d_voxel = DataManager.convert_section_to_z(sec=section, resolution=self.data_feeder.resolution, mid=True, stack=self.gui.stack) d_um = d_voxel * convert_resolution_string_to_voxel_size(stack=self.gui.stack, resolution=self.data_feeder.resolution) polygon.set_properties('position_um', d_um) # print 'd_voxel', d_voxel, 'position_um', d_um else: polygon.set_properties('voxel_position', index) d_um = index * convert_resolution_string_to_voxel_size(stack=self.gui.stack, resolution=self.data_feeder.resolution) polygon.set_properties('position_um', d_um) # print 'index', index, 'position_um', d_um for key, value in kwargs.iteritems(): polygon.set_properties(key, value) return polygon
from flask import Flask, request, render_template, redirect, make_response from werkzeug import secure_filename from data_manager import DataManager from model.puzzle_manager import PuzzleManager from model.puzzle import Puzzle, PuzzleState from model.problem import Problem from model.problem_attempt import ProblemAttempt ########## Initialization ########### app = Flask(__name__, static_url_path="/content", static_folder = "content") datamgr = DataManager() datamgr.load() puzzmgr = PuzzleManager(datamgr) puzzmgr.load() ADMIN_PASSWORD = '******' # Setup web app configurations app.config['SOLUTION_FOLDER'] = 'solutions/' if not os.path.exists(app.config['SOLUTION_FOLDER']): os.makedirs(app.config['SOLUTION_FOLDER']) app.config['MAX_CONTENT_LENGTH'] = 10 * 1024 # 10KB
from flight_data import FlightData from data_manager import DataManager from flight_search import FlightSearch from notification_manager import NotificationManager flight_search = FlightSearch() data_manager = DataManager() sheet_data = data_manager.get_destination_data() if sheet_data[0]["iataCode"] == "": for row in sheet_data: row["iataCode"] = flight_search.get_destination_code(row["city"]) print(sheet_data) data_manager.destination_data = sheet_data data_manager.update_destination_codes() print(sheet_data) cities = [value["iataCode"] for value in sheet_data] for i in cities: x = 0 price = flight_search.get_prices(i) notification = NotificationManager(price.price, price.origin_city, price.origin_airport, price.destination_city, price.return_date, price.out_date, price.destination_airport, price.noinfo) if price.noinfo and price.price < sheet_data[x]["lowestPrice"]:
def __init__( self, batch_size, model_path, examples_path, max_image_width, train_test_ratio, restore, char_set_string, ): self.step = 0 self.CHAR_VECTOR = char_set_string self.NUM_CLASSES = len(self.CHAR_VECTOR) + 1 print(f"CHAR_VECTOR {self.CHAR_VECTOR}") print(f"NUM_CLASSES {self.NUM_CLASSES}") self.__model_path = model_path self.__save_path = os.path.join(model_path, "ckp") self.__restore = restore self.__training_name = str(int(time.time())) self.__session = tf.Session() # Building graph with self.__session.as_default(): ( self.__inputs, self.__targets, self.__seq_len, self.__logits, self.__decoded, self.__optimizer, self.__acc, self.__cost, self.__max_char_count, self.__init, ) = self.crnn(max_image_width) self.__init.run() with self.__session.as_default(): self.__saver = tf.train.Saver(tf.global_variables(), max_to_keep=10) # Loading last save if needed if self.__restore: print("Restoring") ckpt = tf.train.latest_checkpoint(self.__model_path) if ckpt: print("Checkpoint is valid") self.step = int(ckpt.split("-")[1]) self.__saver.restore(self.__session, ckpt) # Creating data_manager self.__data_manager = DataManager( batch_size, model_path, examples_path, max_image_width, train_test_ratio, self.__max_char_count, self.CHAR_VECTOR, )
from data_manager import DataManager from extractors import Extractors from classifiers import Classifiers data_manager = DataManager() extractor = Extractors() classifiers = Classifiers() X, y, encoder = data_manager.loadData() results = [] encoded_x1 = extractor.glcm(X) classifiers.classify(results, encoder, y, title='GLCM - ') encoded_x2 = extractor.lbp(X) classifiers.classify(results, encoder, y, title='LBP - ') encoded_x3 = extractor.huMoments(X) classifiers.classify(results, encoder, y, title='HUMomments - ') for index, result in enumerate(encoded_x1): new_result = result for x2 in encoded_x2[index]: new_result.append(x2) for x3 in encoded_x3[index]: new_result.append(x3) results.append(new_result) classifiers.classify(results, encoder, y, title='Mix - ')
def main(): # create variale scope with tf.variable_scope("NN", reuse=tf.AUTO_REUSE): dm = DataManager(data_dir='data', stopwords_file='', sequence_len=100, test_size=0.2, val_samples=10, n_samples=None, random_state=None) nn = NeuralNetwork(hidden_size=[75], vocab_size=dm.vocab_size, embedding_size=300, max_length=dm.sequence_len, learning_rate=0.01, n_classes=2, random_state=None) # Prepare summaries summaries_dir = '{0}/{1}'.format( "logs", datetime.datetime.now().strftime('%d_%b_%Y-%H_%M_%S')) train_writer = tf.summary.FileWriter(summaries_dir + '/train') validation_writer = tf.summary.FileWriter(summaries_dir + '/validation') # Prepare model directory model_name = str(int(time.time())) model_dir = '{0}/{1}'.format("checkpoint", model_name) if not os.path.exists(model_dir): os.makedirs(model_dir) # Train model init = tf.global_variables_initializer() sess = tf.Session() sess.run(init) saver = tf.train.Saver() x_val, y_val, val_seq_len = dm.get_val_data() train_writer.add_graph(nn.input.graph) for i in range(10): # Perform training step x_train, y_train, train_seq_len = dm.next_batch(10) #print (x_train.shape, y_train.shape, train_seq_len) #(10, 23) (10, 2) [10 6 7 5 7 8 4 5 11 6] train_loss, _, summary = sess.run( [nn.loss, nn.train_step, nn.merged], feed_dict={ nn.input: x_train, nn.target: y_train, nn.seq_len: train_seq_len, nn.dropout_keep_prob: 0.5 }) train_writer.add_summary( summary, i) # Write train summary for step i (TensorBoard visualization) print('{0}/{1} train loss: {2:.4f}'.format(i + 1, "10 epoch", train_loss)) # Check validation performance if (i + 1) % 10 == 0: val_loss, accuracy, summary = sess.run( [nn.loss, nn.accuracy, nn.merged], feed_dict={ nn.input: x_val, nn.target: y_val, nn.seq_len: val_seq_len, nn.dropout_keep_prob: 1 }) validation_writer.add_summary( summary, i ) # Write validation summary for step i (TensorBoard visualization) print(' validation loss: {0:.4f} (accuracy {1:.4f})'.format( val_loss, accuracy)) # Save model checkpoint_file = '{}/model.ckpt'.format(model_dir) save_path = saver.save(sess, checkpoint_file) print('Model saved in: {0}'.format(model_dir))
from data_manager import DataManager from flight_search import FlightSearch from notification_manager import NotificationManager from flight_data import FlightData data = DataManager() flight = FlightSearch() notif = NotificationManager(data) print("Welcome to Sidi Flight Club.") print("We find the best flight deals and email you.") first_name = input("What is your first name?"\n) last_name = input("What is your last name?\n") email = input("What is your email?\n") if email != input("Type your email again.\n"): print("bad email") else: print("You're in the club!") data.add_user(first_name, last_name, email) flight_data = FlightData(data_manager=data, flight_search=flight, notif_manager=notif)
from datetime import datetime, timedelta from data_manager import DataManager from flight_search import FlightSearch from notification_manager import NotificationManager data_manager = DataManager() sheet_data = data_manager.get_destination_data() print(sheet_data) flight_search = FlightSearch() notification_manager = NotificationManager() ORIGIN_CITY_IATA = "OAK" if sheet_data[0]["iataCode"] == "": for row in sheet_data: row["iataCode"] = flight_search.get_destination_code(row["city"]) data_manager.destination_data = sheet_data data_manager.update_destination_codes() tomorrow = datetime.now() + timedelta(days=1) six_month_from_today = datetime.now() + timedelta(days=(6 * 30)) for destination in sheet_data: flight = flight_search.check_flights(ORIGIN_CITY_IATA, destination["iataCode"], from_time=tomorrow, to_time=six_month_from_today) try: if int(flight.price) < int(destination["lowestPrice"]): print(int(flight.price) < int(destination["lowestPrice"])) id = destination["id"]
class DOBService(object): def __init__(self): self.interval = 5.0 self.running = False self.configParser = ConfigParser.RawConfigParser() self.configParser.read('config/dob.cfg') self.data_manager = DataManager(configParser=self.configParser) self.drive_watcher = GDriveWatcher(configParser=self.configParser, data_manager=self.data_manager) self.slack_bot = SlackBot(configParser=self.configParser) self.witmanager = WitAIManager(configParser=self.configParser) self.last_drive_scan = None def start(self): if self.running: return logging.info('Service starting') self.setup() self.running = True # self.thread = threading.Thread(target=self.run, args=()) # self.thread.daemon = True # self.thread.start() self.run() def setup(self): self.data_manager.connect() self.drive_watcher.connect() self.slack_bot.connect() def run(self): while self.running: now = datetime.datetime.now() # get updated files if self.last_drive_scan is None or ((now - self.last_drive_scan).seconds * 0.0166667) >= 30: self.last_drive_scan = now updated_files = self.drive_watcher.update() self.broadcast_updated_files(updated_files) self.process_slack_messages() time.sleep(self.interval) def process_slack_messages(self): packets = self.slack_bot.update() for packet in packets: event_type = packet['type'] #if event_type == 'hello': #elif event_type == 'message': # self.process_slack_message(packet['message']) #elif event_type == 'user_typing': # #elif event_type == 'team_join': # #elif event_type == 'presence_change': # #else: # if event_type == 'message': self.process_slack_message(packet) def process_slack_message(self, packet): if packet is None or len(packet) == 0: return logging.info('Processing message {}'.format(packet)) if not self.is_valid_slack_message(packet): return user = self.slack_bot.users[packet['user']] channel = self.slack_bot.channels[self.slack_bot.get_channel_key_for_channel_with_id(packet['channel'])] text = packet['text'] if text is None or len(text) == 0: return import re matches = re.findall ('<@[a-zA-z0-9]*>:', text, re.DOTALL) idx = 0 for match in matches: #if ":" in match: if idx == 0: text = text.replace(match, "") else: user_id = match.replace("<", "").replace(">", "").replace("@", "").replace(":", "").strip() user = None if user_id in self.slack_bot.users: user = self.slack_bot.users[user_id] if user is not None: #text = text.replace(match, user['emailAddress']) text = text.replace(match, user['name']) else: text = text.replace(match, "") idx += 1 status, outcomes = self.witmanager.query(text) if outcomes is None or len(outcomes) == 0: return outcome = outcomes[0] # TODO: check that the first always has the highest confience value logging.info('Message has intent {}'.format(outcome.intent)) if outcome.intent == WitAIOutcome.INTENT_RETRIEVE_FILES: self.process_intent_to_retrieve_files(user, channel, outcome) def process_intent_to_retrieve_files(self, user, channel, outcome): min_date = None max_date = None contacts = [] for entity in outcome.entities: if entity.type == WitAIEntity.TYPE_CONTACT: if entity.value is None or len(entity.value) == 0: continue contacts.append(entity.value.lower()) elif entity.type == WitAIEntity.TYPE_DATE: cur_date = entity.value_to_date() if cur_date is not None: if min_date is None and max_date is None: min_date = cur_date max_date = cur_date else: if min_date > cur_date: min_date = cur_date if max_date < cur_date: max_date = cur_date # query db entities = self.data_manager.search_files(contacts=contacts, min_date=min_date, max_date=max_date) if entities is None or len(entities) == 0: msg = "@{} cannot find any files based on your search terms".format(user['name']) self.slack_bot.send_message(msg, channel=channel['id']) else: #msgs = [] msg = "here are some files matching your query:" for entity in entities: if entity.mimeType == GMimeType.Folder: continue #msgs.append("{} ({})".format(entity.title, entity.get_link())) msg += "\n" link = entity.get_link() if link is None or len(link) == 0: msg += "{}".format(entity.title) else: msg += "{} ({})".format(entity.title, entity.get_link()) #for msg in msgs: # self.slack_bot.send_message(msg, channel=channel['id']) self.slack_bot.send_message(msg, channel=channel['id']) def is_valid_slack_message(self, message): if 'channel' not in message: return False if 'text' not in message: return False channel = message['channel'] text = message['text'] mentions = DOBService.extract_entities_from_text(text) return self.slack_bot.me['user_id'] in mentions and channel == self.slack_bot.channels[self.slack_bot.channel]['id'] @staticmethod def extract_entities_from_text(text): import re return re.findall('<@(.*?)>', text, re.DOTALL) def broadcast_updated_files(self, updated_files): print "broadcast_updated_files; updated/added count {}".format(len(updated_files)) for entity in updated_files: msgs = self.construct_slack_message_for_entity(entity) for msg in msgs: self.slack_bot.send_message(msg) def construct_slack_message_for_entities(self, updated_files): now = datetime.datetime.now() updates_dict = {} msgs = [] for entity in updated_files: if (now - entity.modifiedDate).days >= 2: print "ignoring {}".format(entity.title) continue title = entity.title link = entity.get_link() if entity.createdDate == entity.modifiedDate: key = entity.get_owners() else: key = entity.get_modifier() if key not in updates_dict: updates_dict[key] = [] updates_dict[key].append((title, link)) # create messages for key, value in updates_dict.iteritems(): titles = value[0] links = value[1] if len(titles) == 1: msg = "{} updated {} ({})".format(key, titles[0], links[0]) else: msg = "{} has updated " for i in range(len(titles)): if i > 0: msg += ", " msg += titles[i] msg += " ({})".format(links[i]) msgs.append(msg) return msgs def construct_slack_message_for_entity(self, entity): now = datetime.datetime.now() if (now - entity.modifiedDate).days >= 2: print "ignoring {}".format(entity.title) return [] if entity.createdDate == entity.modifiedDate: return ["{} has created a new file {}\n{}".format( entity.get_owners(), entity.title, entity.get_link())] else: return ["{} has modified {}\n{}".format( entity.get_modifier(), entity.title, entity.get_link())] def stop(self): self.running = False
def __init__(self, server=requests): self.server = server self.data = DataManager(self.server)
def setUp(self): self.data_manager = DataManager()
import os from data_manager import DataManager manager = DataManager() a = manager.get_contests() b = manager.get_tasks() print a[0].key print a[0].short_name print a[0].remaining print a[0].organizer print a[0].full_name print a[0].year print a[0].round print a[0].num_tasks print a[0].url print '' print a[-1].key print a[-1].short_name print a[-1].remaining print a[-1].organizer print a[-1].full_name print a[-1].year print a[-1].round print a[-1].num_tasks print a[-1].url print '' print b[0].contests_key print b[0].name
def get_pos_tags(): words = DataManager.get_sentences_clean() get_pos_tag_comments(words, '../features/pos_tags.csv') get_n_grams(3, '../features/pos_tags.csv', '../features/pos_ngrams.csv') get_ids('../features/pos_ngrams.csv')
def __init__(self): self.data_manager = DataManager()
class Main(object): def __init__(self, top_right, bottom_left): self.event_instance = None self.data_manager = DataManager() self.top_right = top_right self.bottom_left = bottom_left self.result = self.run() self.data_manager.close_con() def get_result(self): return self.result def run(self): self.event_instance = self.data_manager.get_events_for_view(self.top_right, self.bottom_left) train_and_test = self.divide_events_into_train_test() training_data_array = np.array(self.event_instance.get_train_list(train_and_test["training_labels"])) training_target_array = np.array(self.event_instance.get_target_list(train_and_test["training_labels"])) testing_array = np.array(self.event_instance.get_test_list(train_and_test["testing_labels"])) testing_target = np.array(self.event_instance.get_target_list(train_and_test["testing_labels"])) #knn = neighbors.KNeighborsClassifier(10) #knn.fit(training_data_array, training_target_array) #testing_target = knn.predict(testing_array) data_points = np.concatenate((training_data_array, testing_array)) targets = np.concatenate((training_target_array, testing_target)) knn = neighbors.KNeighborsClassifier(12) knn.fit(data_points, targets) targets = knn.predict(data_points) targets_map = list() target_index = 0 for target in np.unique(targets): targets_map.append((target, target_index, Event.get_category_color(target))) target_index += 1 result = [] result.append([[t, c] for (t, i, c) in targets_map]) dp = list() #df = file("datapoints.csv", "w") for i in range(len(data_points)): # df.write(str(data_points[i][0])+","+str(data_points[i][1])+","+str(targets_map[targets[i]][0])+"\n") index = [index for (t, index, c) in targets_map if t == targets[i]] dp.append([data_points[i][0], data_points[i][1], index[0]]) result.append(dp) return result #df.close() #tf = file("targets.csv", "w") #for i in range(len(targets)): # tf.write(str(targets_map[targets[i]])+","+targets[i]+"\n") #tf.close() def divide_events_into_train_test(self): frequencies = Counter(self.event_instance.get_categories()) total = sum(frequencies.values()) for key in frequencies: frequencies[key] /= float(total) values = sorted(frequencies.values(), reverse=True) if len(values) > 4: values = values[0:5] train_labels = [k for (k, v) in frequencies.items() if frequencies[k] in values and frequencies[k] > .05] other_labels = [k for (k, v) in frequencies.items() if k not in train_labels] else: train_labels = [k for (k, v) in frequencies.items() if frequencies[k] > .05] other_labels = [k for (k, v) in frequencies.items() if k not in train_labels] return {"training_labels": train_labels, "testing_labels": other_labels}
def validation(project_name, sliding_window_size, check_days, max_k, random_val): """ Perform validation with given parameters. Parameters ---------- project_name (str): Name of the project to read change sets. sliding_window_size (str): Number of days to include the graph. check_days (list): List of integers to check if recomendations are true or false. max_k (int): Maximum k for topk and MRR calculations. When max_k is 3, top1, top2 and top3 will be calculated, and the ranks in MRR calculations can 1, 2 and 3. random_val (bool): If True, `max_k` replacements will be selected randomly. Returns ------- list: First item of the list is the name of the experiment. Second and the following items will include accuracy and MRR for each check day. For example, returns [pig_sws365, (7, {top1:.5, top2:.7, mrr:.6}), (30, {top1:.6, top2:.9, mrr:.7})]. """ dataset_path = get_dataset_path(project_name) exp_name = get_exp_name(project_name, sws=sliding_window_size) dm = DataManager(dataset_path, None) # No need for sliding window size G = HistoryGraph(dataset_path, sliding_window_size) check_day_to_ranks = {check_day: [] for check_day in check_days} date_to_results = load_results(exp_name) for date, results in date_to_results.items(): if not results["replacements"]: # No leaving developer continue G.forward_until(date) # Update graph for leaving_dev, recommended_devs in results["replacements"].items(): if not recommended_devs: # No recommended developers continue if random_val: # Randomly select "max_k" developers other_devs = results["developers"] other_devs.remove(leaving_dev) recommended_devs = random.sample(other_devs, max_k) else: # Convert dictionary keys to list and get first "max_k" items recommended_devs = list(recommended_devs)[:max_k] leaving_dev_files = set(G.find_reachable_files(leaving_dev)) for check_day in check_days: # Get the change sets in the next days # For example, get the change sets in the next 7 days if check day is 7 change_sets = dm.get_specific_window( date + timedelta(days=1), date + timedelta(days=check_day)) rank = float("inf") # Not found yet for i, recommended_dev in enumerate(recommended_devs): recommended_dev_files = set( G.find_reachable_files(recommended_dev)) # Find the files that leaving developer can reach but recmommended # developer cannot reach target_files = leaving_dev_files - recommended_dev_files if check_modification(change_sets, recommended_dev, target_files): rank = i + 1 break # No need to check other developers check_day_to_ranks[check_day].append(rank) ret_items = [exp_name] for check_day in check_days: res = {} for k in range(1, max_k + 1): res["top{}".format(k)] = cal_accuracy( check_day_to_ranks[check_day], k) res["mrr"] = cal_mrr(check_day_to_ranks[check_day]) ret_items.append((check_day, res)) return ret_items
# stack = args.stack elastix_output_dir = args.elastix_output_dir custom_output_dir = args.custom_output_dir image_name_list = load_txt(args.image_name_list) toanchor_transforms_fp = args.toanchor_transforms_fp ################################################# anchor_idx = image_name_list.index(args.anchor_img_name) transformation_to_previous_sec = {} for i in range(1, len(image_name_list)): transformation_to_previous_sec[i] = DataManager.load_consecutive_section_transform(moving_fn=image_name_list[i], fixed_fn=image_name_list[i-1], elastix_output_dir=elastix_output_dir, custom_output_dir=custom_output_dir) transformation_to_anchor_sec = {} for moving_idx in range(len(image_name_list)): if moving_idx == anchor_idx: # transformation_to_anchor_sec[moving_idx] = np.eye(3) transformation_to_anchor_sec[image_name_list[moving_idx]] = np.eye(3) elif moving_idx < anchor_idx: T_composed = np.eye(3) for i in range(anchor_idx, moving_idx, -1): T_composed = np.dot(np.linalg.inv(transformation_to_previous_sec[i]), T_composed) # transformation_to_anchor_sec[moving_idx] = T_composed transformation_to_anchor_sec[image_name_list[moving_idx]] = T_composed
""" @author: isabelleguyon This is an example of program that tests the Iris challenge Preprocessor class. Another style is to incorporate the test as a main function in the Data manager class itself. """ from sys import path path.append("../ingestion_program") # Contains libraries you will need from data_manager import DataManager # such as DataManager from prepro import Preprocessor input_dir = "../sample_data" output_dir = "../resuts" basename = 'credit' D = DataManager(basename, input_dir) # Load data print("*** Original data ***") print D Prepro = Preprocessor() # Preprocess on the data and load it back into D D.data['X_train'] = Prepro.fit_transform(D.data['X_train'], D.data['Y_train']) D.data['X_valid'] = Prepro.transform(D.data['X_valid']) D.data['X_test'] = Prepro.transform(D.data['X_test']) # Here show something that proves that the preprocessing worked fine print("*** Transformed data ***") print D # Preprocessing gives you opportunities of visualization:
# Put here your OWN test code # To make sure this runs on Codalab, put here things that will not be executed on Codalab from sys import argv, path path.append ("../starting_kit/ingestion_program") # Contains libraries you will need from data_manager import DataManager # such as DataManager if len(argv)==1: # Use the default input and output directories if no arguments are provided input_dir = "../public_data" # Replace by correct path output_dir = "../results" # Create this directory if it does not exist else: input_dir = argv[1] output_dir = argv[2]; basename = 'cifar10' D = DataManager(basename, input_dir) # Load data print("*** Original data ***") print D Prepro = PCAPreprocessor() # Preprocess on the data and load it back into D D.data['X_train'] = Prepro.fit_transform(D.data['X_train'], D.data['Y_train']) # Les donnes sont bien de dimensions 192 D.data['X_valid'] = Prepro.transform(D.data['X_valid']) D.data['X_test'] = Prepro.transform(D.data['X_test']) # Here show something that proves that the preprocessing worked fine print("*** Transformed data ***") print D
if row != row_size - 1: row_images_with_spacers.append(spacer_h) ret = np.vstack(row_images_with_spacers) return ret def convert_to_colomap(im, cmap): im = cmap(im) im = np.uint8(im * 255) return im np.random.seed(1) data_manager = DataManager() place_cells = PlaceCells() hd_cells = HDCells() data_manager.prepare(place_cells, hd_cells) model = Model(place_cell_size=place_cells.cell_size, hd_cell_size=hd_cells.cell_size, sequence_length=100) sess = tf.Session() sess.run(tf.global_variables_initializer()) # Load checkpoints load_checkpoints(sess)
def setUp(self): self.manager = DataManager() self.manager.load()
# ================ @CODE SUBMISSION ================= overall_time_budget = 0 time_left_over = 0 for basename in datanames: # Loop over datasets vprint( verbose, "************************************************") vprint( verbose, "******** Processing dataset " + basename.capitalize() + " ********") vprint( verbose, "************************************************") # ======== Learning on a time budget: # Keep track of time not to exceed your time budget. Time spent to inventory data neglected. start = time.time() # ======== Creating a data object with data, informations about it vprint( verbose, "========= Reading and converting data ==========") D = DataManager(basename, input_dir, replace_missing=True, filter_features=True, max_samples=max_samples, verbose=verbose) print (D) vprint( verbose, "[+] Size of uploaded data %5.2f bytes" % data_io.total_size(D)) # ======== Keeping track of time if debug_mode<1: time_budget = D.info['time_budget'] # <== HERE IS THE TIME BUDGET! else: time_budget = max_time overall_time_budget = overall_time_budget + time_budget vprint( verbose, "[+] Cumulated time budget (all tasks so far) %5.2f sec" % (overall_time_budget)) # We do not add the time left over form previous dataset: time_budget += time_left_over vprint( verbose, "[+] Time budget for this task %5.2f sec" % time_budget) time_spent = time.time() - start vprint( verbose, "[+] Remaining time after reading data %5.2f sec" % (time_budget-time_spent)) if time_spent >= time_budget:
import os from data_manager import DataManager manager = DataManager() a = manager.get_contests() b = manager.get_tasks() print a[0].key print a[0].short_name print a[0].remaining print a[0].organizer print a[0].full_name print a[0].year print a[0].round print a[0].num_tasks print a[0].url print '' print a[-1].key print a[-1].short_name print a[-1].remaining print a[-1].organizer print a[-1].full_name print a[-1].year print a[-1].round print a[-1].num_tasks print a[-1].url print '' print b[0].contest_short_name print b[0].name
plt.plot() if __name__ == "__main__": # We can use this to run this file as a script and test the Preprocessor if len( argv ) == 1: # Use the default input and output directories if no arguments are provided input_dir = "public_data" output_dir = "results" else: input_dir = argv[1] output_dir = argv[2] basename = 'plankton' D = DataManager(basename, input_dir) # Load data print("*** Original data ***") print(D) Prepro = Preprocessor() # Preprocess on the data and load it back into D D.data['X_train'] = Prepro.fit_transform(D.data['X_train'], D.data['Y_train']) D.data['X_train'], D.data['Y_train'] = Prepro.removeOutliers( D.data['X_train'], D.data['Y_train']) D.data['X_valid'] = Prepro.transform(D.data['X_valid']) D.data['X_test'] = Prepro.transform(D.data['X_test']) D.feat_name = np.array(['PC1', 'PC2']) D.feat_type = np.array(['Numeric', 'Numeric'])
class Evaluation: def __init__(self, path): print("Evaluation") self.path = path self.path_logs = os.path.join(path, "logs") self.data_manager = DataManager(path + 'data/mnist/', batch_size=1) def create_graph(self): # TODO: get_Variable 써보기? x = tf.placeholder(tf.float32, shape=[None, 28, 28, 1]) conv1 = conv_layer(x, shape=[5, 5, 1, 32]) conv1_pool = max_pool_2x2(conv1) conv2 = conv_layer(conv1_pool, shape=[5, 5, 32, 64]) conv2_pool = max_pool_2x2(conv2) conv2_flat = tf.reshape(conv2_pool, [-1, 7 * 7 * 64]) full_1 = tf.nn.relu(full_layer(conv2_flat, 1024)) keep_prob = tf.placeholder(tf.float32) full1_drop = tf.nn.dropout(full_1, keep_prob=keep_prob) y_conv = full_layer(full1_drop, 10) return x, keep_prob, y_conv def create_optimizor(self, y_conv): y_ = tf.placeholder(tf.float32, shape=[None, 10]) cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y_conv, labels=y_)) train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) return y_, train_step, accuracy def run_evaluation(self): # ------------ train.py의 run_training()와 동일 ------------------ # 그래프 구성하기 x, keep_prob, y_conv = self.create_graph() # 최적화 그래프 만들기(loss function, optimizer, accuracy evaluator) y_, train_step, accuracy = self.create_optimizor(y_conv) # -------------------------------------------------------------- # Saver 만들기 saver = tf.train.Saver() # 세션 열기 with tf.Session() as sess: # 저장된 model-XXXX checkpoint로 모델 불러오기 saver.restore(sess, os.path.join(self.path_logs, "model-1000")) # 테스트를 위한 데이터 불러오기 X, Y = self.data_manager.get_test_batch_data() # self.data_manager.get_test_data() # 테스트 수행 test_accuracy = sess.run(accuracy, feed_dict={x: X, y_: Y, keep_prob: 1.0}) print("Accuracy: {}".format(test_accuracy))
def setUp(self): self.manager = DataManager()
def __init__(self, path): print("Evaluation") self.path = path self.path_logs = os.path.join(path, "logs") self.data_manager = DataManager(path + 'data/mnist/', batch_size=1)