class MainWindow: def __init__(self): self.main_win = QMainWindow() self.ui = Ui_MainWindow() self.ds = DataStore() self.ui.setupUi(self.main_win) self.ui.stackedWidget.setCurrentWidget(self.ui.home) self.ui.bmr_batton.clicked.connect(self.showBMR) self.ui.bmi_button.clicked.connect(self.showBMI) self.ui.bmr_calculate_button.clicked.connect(self.calcBMR) self.ui.bmi_calculate_button.clicked.connect(self.calcBMI) def show(self): self.main_win.show() def showBMR(self): self.ui.stackedWidget.setCurrentWidget(self.ui.bmr) def showBMI(self): self.ui.stackedWidget.setCurrentWidget(self.ui.bmi) def calcBMR(self): if self.ui.bmr_male_radio.isChecked(): gender = "male" else: gender = "female" height = int(self.ui.bmr_height_input.text()) weight = int(self.ui.bmr_weight_input.text()) age = int(self.ui.bmr_age_input.text()) if self.ui.bmr_sedentary.isChecked(): activity = "sedentary" elif self.ui.bmr_lighthy.isChecked(): activity = "lighthy" elif self.ui.bmr_moderately.isChecked(): activity = "moderately" elif self.ui.bmr_very.isChecked(): activity = "very" else: activity = "extremely" result = self.ds.bmr_calc(gender, height, weight, age, activity) output = "The BMR for a {} year old {}, who is {} cm tall and {} kg heavy with a {} activity level is {} calories.".format( age, gender, height, weight, activity, result) self.ui.bmr_output_label.setText(output) def calcBMI(self): height = int(self.ui.bmi_height_input.text()) weight = int(self.ui.bmi_weight_input.text()) result = self.ds.bmi_calc(height, weight) outpuut = "You entered the following information for BMI: Height {} cm & Weight: {} kg. This meansyourBMI is {}".format( height, weight, result) self.ui.bmi_output_label.setText(outpuut)
def __init__(self, str_id, df=None, rematch=False): self.df = df self.str_id = str_id self.initialize() self.ds = DataStore() if rematch: self.ds.rematch = True
def __init__(self): self.main_win = QMainWindow() self.ui = Ui_MainWindow() self.ds = DataStore() self.ui.setupUi(self.main_win) self.ui.stackedWidget.setCurrentWidget(self.ui.home) self.ui.bmr_batton.clicked.connect(self.showBMR) self.ui.bmi_button.clicked.connect(self.showBMI) self.ui.bmr_calculate_button.clicked.connect(self.calcBMR) self.ui.bmi_calculate_button.clicked.connect(self.calcBMI)
def __init__(self, rand_seed, display=False, no_op_max=27, testing=False, show_trades=None): #self.ale = ALEInterface() np.random.seed(rand_seed) self._no_op_max = no_op_max self.sequence_length = SEQUENCE_LENGTH features_list = FEATURES_LIST self.features_length = len(FEATURES_LIST) path = STORE_PATH training_days = TRAINING_DAYS testing_days = TESTING_DAYS # Load the data training_store = DataStore(training_days=training_days, features_list=features_list, sequence_length=self.sequence_length) if testing: print("Set up for testing") testing_store = DataStore(training_days=training_days, testing_days=testing_days, features_list=features_list, sequence_length=self.sequence_length, mean=training_store.mean, std=training_store.std) self.environment = Trading(data_store=testing_store, sequence_length=self.sequence_length, features_length=self.features_length, testing=testing, show_trades=show_trades) else: self.environment = Trading(data_store=training_store, sequence_length=self.sequence_length, features_length=self.features_length, testing=testing, show_trades=show_trades) self.old_x = 0. # collect minimal action set #self.real_actions = self.ale.getMinimalActionSet() self.real_actions = [0, 1, 2] # height=210, width=160 self.reset()
def __init__(self, rand_seed, display=False, no_op_max=27, testing=False, show_trades=None): np.random.seed(rand_seed) self._no_op_max = no_op_max self.sequence_length = SEQUENCE_LENGTH features_list = FEATURES_LIST self.features_length = len(FEATURES_LIST) path = STORE_PATH training_days = TRAINING_DAYS testing_days = TESTING_DAYS # Load the data training_store = DataStore(training_days=training_days, sequence_length=self.sequence_length) if testing: print("Set up for testing") testing_store = DataStore(training_days=training_days, testing_days=testing_days, sequence_length=self.sequence_length, mean=training_store.mean, std=training_store.std) self.environment = Trading(data_store=testing_store, sequence_length=self.sequence_length, features_length=self.features_length, testing=testing, show_trades=show_trades) else: self.environment = Trading(data_store=training_store, sequence_length=self.sequence_length, features_length=self.features_length, testing=testing, show_trades=show_trades) self.old_x = 0. self.reset()
def randomize(graph): i = 0 for n in graph.nodes(): i += 1 randomloc = float(random.randint(0, 999999999)) / 1000000000 newnode = (randomloc, ) neighbors = graph.neighbors(n) graph.add_node(newnode, id=i, ds=DataStore(100000)) for neighbor in neighbors: graph.add_edge(newnode, neighbor) graph.remove_node(n)
def choose_file_rating(folder="data/match_rating/"): ds = DataStore() files = os.listdir(folder) files.sort(key=lambda f: int(re.sub('\D', '', f))) print("Welcome to the analyzer. Which file would you like to analyze?") for index, file in enumerate(files): print("({}) {}".format(index, file)) file_index = input( "Write the number to choose the file. Write anything else to cancel\n") if int(file_index) in range(1, len(files)): print("File {} chosen".format(files[int(file_index)])) results = ds.load_rating_file(files[int(file_index)]) return results else: print("Command not recognized. Returning.") return
def calculate_all_ratings(folder="data/match_rating/"): ds = DataStore() files = os.listdir(folder) files.sort(key=lambda f: int(re.sub('\D', '', f))) for file in files: print("File {} chosen".format(file)) results = ds.load_rating_file(file) # calculate the relevant data count_maybe = 0 count_yes = 0 count_no = 0 for i, result in results.iterrows(): rating = result["match_rating"] if rating == 0: count_no += 1 elif rating == 1: count_yes += 1 elif rating == -1: count_maybe += 1 # Create data data = {"bad": count_no, "good": count_yes, "maybe": count_maybe} output_path = "data/rating_result.csv" now_id = file.replace("rating_", "").replace(".csv", "") data["id"] = now_id if os.path.exists(output_path): df = pd.read_csv(output_path) df2 = pd.DataFrame([data]) df3 = df.append(df2) df3.to_csv(output_path, index=False, mode='w') else: df = pd.DataFrame([data]) df.to_csv(output_path, index=False, mode='w') print("Good: {}\nBad: {}\nMaybe: {}".format(count_yes, count_no, count_maybe))
def __init__(self): #实例化其他几个功能模块 self.urlgen = URLGenerator() self.downloader = Downloader() self.parser = DataParser() self.datastore = DataStore() #建立3个url队列:未下载、正在下载、完成下载 self.beforedownloadset = set() self.beingdownloadset = set() self.afterdownloadset = set() #设定种子url self.seedurl = 'https://so.gushiwen.org/authors/'
def get_risk_map(self, location=None, box=None, name="map", accuracy=10): if location is not None: locs = location.split(',') resp = requests.get(self._BASE_URL.format( locs[0], locs[1])).json()["boundingbox"] box = resp[0] + "," + resp[2] + "," + resp[1] + "," + resp[3] coords = str(box).split(',') first = Location(float(coords[0]), float(coords[1])) second = Location(float(coords[2]), float(coords[3])) locParser = LocationParser(first, second, accuracy=accuracy) locParser.parse() elevParser = ElevationParser(locParser.grid) DataStore.save(elevParser.grid, name=name) data = DataStore.load(name) s = getWatershed(data, 100) m = RiskMap(s, 10) m.generate_kml(name=name)
def main(): if (len(sys.argv) > 1): if sys.argv[1] == "check": idl_path = parse(sys.argv[2:], 'i:', ["idl="])[0] #To do: call api dds = DDSConnector(idl_path) if dds.IsIDLValid(): print("valid") else: print("invalid") elif sys.argv[1] == "run": config = configparser.ConfigParser() config.read_file(open(parse(sys.argv[2:], 'c:', ["config="])[0])) if config['common']['converter'] == 'dds': if config['dds']['mode'] == 'subscriber': dtStore = DataStore(db_name=config['common']['package_id']) dtStore.AddDataToCollection( "file_idl", json_obj={ "id": dtStore.AddFileToDB(config['dds']['file_idl']) }) dtStore.AddDataToCollection( "network_config", json_obj={ "id": dtStore.AddFileToDB( config['dds']['network_config']) }) ckan = CKANConnector(config['common']['ckan_sever'], config['common']['api_token'], config['common']['resource_id'], config['common']['package_id']) ckan.CreateResource(config['dds']['file_idl'], "The idl file") ckan.CreateResource(config['dds']['network_config'], "The network config file") x = threading.Thread(target=ReceiveStreamData, args=[dtStore, ckan], daemon=True) x.start() dds = DDSConnector(config['dds']['file_idl'], config['dds']['topic_name'], config['dds']['network_config'], config['dds']['mode']) dds.Build() dds.Run() elif sys.argv[1] == "publish_example": DDSConnector().PublishExample() else: print(sys.argv) print(USAGE) else: print(sys.argv) print(USAGE)
def choose_file(folder="data/matching_result/"): ds = DataStore() mentors, mentees, person_dict = ds.load_data("initial_data_{}".format( global_vars.ROUND)) files = os.listdir(folder) files.sort(key=lambda f: int(re.sub('\D', '', f))) print("Welcome to the analyzer. Which file would you like to analyze?") for index, file in enumerate(files): print("({}) {}".format(index, file)) file_index = input( "Write the number to choose the file. Write anything else to cancel\n") if int(file_index) in range(1, len(files)): print("File {} chosen".format(files[int(file_index)])) results = ds.load_result_file(files[int(file_index)]) str_id = files[int(file_index)].replace("matching_result_", "").replace(".csv", "") else: print("Command not recognized. Returning.") return return person_dict, results, str_id
def __init__(self): data_sources = DataSources(authors="data/author.csv", books="data/book.csv", books_to_subjects="data/book2subjects.csv", subjects="data/subject.csv", sub_subjects="data/sub_subject.csv", sub_sub_subjects="data/sub_sub_subject.csv") self._data_store = DataStore(data_sources) self._excel = ExcelReportGenerator(self._data_store) self._controller = Controller(self._data_store, self._excel) self._window = AppWindow(self._controller) sys.exit(self._app.exec_())
def hungarian_main(self): global ds ds = DataStore() mentors, mentees = self.create_assignment_matrix() self.matrix_size = len(self.score_matrix) aux_row_vars = self.init_row_aux_vars() aux_col_vars = self.init_col_aux_vars() # Master method for the hungarian algorithm / job assignment self.hungarian_algorithm(aux_row_vars, aux_col_vars, 1) matching_list = self.format_matching_result(mentors, mentees) # ds.save_matches(matching_list) show_matches(matching_list) return matching_list
for f in next(os.walk(v_path), (None, None, []))[2]: os.remove(os.path.join(v_path, f)) # Training sets """input_text = "abcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddef"\ + "abcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddef""""" #input_text = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, ipsum sed diam" #with os.open("lstm_training_data/vanishing_vocables_de.txt", "r"): #with os.open("lstm_training_data/loremipsum.txt", "r"): with os.fdopen(os.open("lstm_training_data/schachnovelle.txt", os.O_RDONLY)) as f: input_text = f.read() # initialize data store object data_store = DataStore( input_data_set=set(input_text), output_data_set=set(input_text), input_data=input_text) # set data_store configuration data_store.configure({ "sequence_length": config["time_steps"] }) # samples is a generator object for sequences of length data_store.config['time_steps'] samples = data_store.samples() # Uncomment to use random samples for training. The amount specifies the number of sequences # to be trained during one iteration. After each iteration the sequences will be shuffled again. #samples = data_store.random_samples(amount=20) lstm = LSTMNetwork()
def __init__(self): value = { "Command": "INIT", "Token": "8051bf89-e115-4147-8e5a-ff9d6f39f0d7" } headers = { 'Content-type': 'application/json', 'Accept': 'application/json', } jvalue = json.dumps(value) conn = httplib.HTTPConnection('107.20.243.77', 80) conn.request('POST', '/api/hermes', jvalue, headers) response = conn.getresponse() ret = json.loads(str((response.status, response.reason, response.read())[2])) conn.close() DS = DataStore() ctrl = controller() myFile = open('output.txt', 'w') myFile.write(str(ret)) myFile.close() looptime = ret['ServerState']['ServerTiers']['DB']['ServerStartTurnTime'] coef = (ret["ServerState"]["CostPerServer"] / ret["ServerState"]["ProfitConstant"]) DS.setCoef(coef) infra = False p = None research = None didGrid = False progression = [None, "GRID", "GREEN", None ] #while ret['ServerState']['TurnNo'] < 10080: while True: x = 0 while x <=looptime-1: value = { "Command": "PLAY", "Token": "8051bf89-e115-4147-8e5a-ff9d6f39f0d7" } headers = { 'Content-type': 'application/json', 'Accept': 'application/json', } jvalue = json.dumps(value) conn = httplib.HTTPConnection('107.20.243.77', 80) conn.request('POST', '/api/hermes', jvalue, headers) response = conn.getresponse() ret = json.loads(str((response.status, response.reason, response.read())[2])) #Get demand from server demand = [ret['ServerState']['ServerTiers']['DB']['ServerRegions']['NA']['NoOfTransactionsInput']] demand.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['EU']['NoOfTransactionsInput']) demand.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['AP']['NoOfTransactionsInput']) config = [ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['NA']['NodeCount']] config.append(ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['EU']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['AP']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['JAVA']['ServerRegions']['NA']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['JAVA']['ServerRegions']['EU']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['JAVA']['ServerRegions']['AP']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['NA']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['EU']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['AP']['NodeCount']) DS.avgDemand(demand) DS.setConfig(config) conn.close() x+=1 lastProfit = ret["ServerState"]["ProfitEarned"] GridCost = ret["ServerState"]["ResearchUpgradeLevels"][1]["UpgradeCost"] GridTurns = ret["ServerState"]["ResearchUpgradeLevels"][1]["NoOfTurnsRequired"] GridTotalCost = GridTurns * GridCost if ret['ServerState']['TurnNo'] <= 9000 and ret["ServerState"]["ProfitAccumulated"] >= GridTotalCost/10 and GridCost < (lastProfit - (lastProfit/3)): didGrid = True try: if ret["ServerState"]["ResearchUpgradeState"]["GRID"] == -1: #research = "GREEN" pass except: research = "GRID" #p = research #Calculate free space #AVERAGE CAPACITY # capacity = [ ( ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2] # capacity.append(( ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2) # capacity.append(( ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2) #97% capacity = [ ( ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']+ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] ) / 3] capacity.append(( ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] ) / 3) capacity.append(( ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit']) / 3) #93% # capacity = [ ( ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']+ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 3] # capacity.append(( ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 3) # capacity.append(( ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']) / 3) #100% CAPACITY # capacity = [ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit']] # capacity.append(ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit']) # capacity.append(ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit']) # #90% CAPACITY # capacity = [ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']] # capacity.append(ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']) # capacity.append(ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']) DS.setCapacity(capacity) changes = ctrl.calc(DS) jsonchange = {"Servers":{ "WEB":{ "ServerRegions":{ "AP":{ "NodeCount":changes[2] }, "EU":{ "NodeCount":changes[1] }, "NA":{ "NodeCount":changes[0] } } }, "JAVA":{ "ServerRegions":{ "NA":{ "NodeCount":changes[3] }, "EU":{ "NodeCount":changes[4] }, "AP":{ "NodeCount":changes[5] } } }, "DB":{ "ServerRegions":{ "NA":{ "NodeCount":changes[6] }, "EU":{ "NodeCount":changes[7] }, "AP":{ "NodeCount":changes[8] } } } }, "UpgradeInfraStructure": infra, "UpgradeToResearch": research } if research != None: research = None value = { "Command": "CHNG", "Token": "8051bf89-e115-4147-8e5a-ff9d6f39f0d7", "ChangeRequest": jsonchange } headers = { 'Content-type': 'application/json', 'Accept': 'application/json', } jvalue = json.dumps(value) conn = httplib.HTTPConnection('107.20.243.77', 80) conn.request('POST', '/api/hermes', jvalue, headers) response = conn.getresponse() ret = json.loads(str((response.status, response.reason, response.read())[2])) # if research != None: # research = None # jsonchange = {"Servers":{ # "WEB":{"UpgradeToResearch": "Green"}}} # value = { # "Command": "CHNG", # "Token": "8051bf89-e115-4147-8e5a-ff9d6f39f0d7", # #"Token": "7440b0b0-c5a2-4ab3-bdc3-8935865bb9d1", # "ChangeRequest": jsonchange # } # headers = {'Content-type': 'application/json','Accept': 'application/json',} # jvalue = json.dumps(value) # conn = httplib.HTTPConnection('107.20.243.77', 80) # #conn = httplib.HTTPConnection('uat.hermes.wha.la', 80) # conn.request('POST', '/api/hermes', jvalue, headers) # response = conn.getresponse() # ret = json.loads(str((response.status, response.reason, response.read())[2])) #play value = { "Command": "PLAY", "Token": "8051bf89-e115-4147-8e5a-ff9d6f39f0d7" } headers = { 'Content-type': 'application/json', 'Accept': 'application/json', } jvalue = json.dumps(value) conn = httplib.HTTPConnection('107.20.243.77', 80) conn.request('POST', '/api/hermes', jvalue, headers) response = conn.getresponse() ret = json.loads(str((response.status, response.reason, response.read())[2])) #Get demand from server demand = [ret['ServerState']['ServerTiers']['DB']['ServerRegions']['NA']['NoOfTransactionsInput']] demand.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['EU']['NoOfTransactionsInput']) demand.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['AP']['NoOfTransactionsInput']) config = [ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['NA']['NodeCount']] config.append(ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['EU']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['AP']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['JAVA']['ServerRegions']['NA']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['JAVA']['ServerRegions']['EU']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['JAVA']['ServerRegions']['AP']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['NA']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['EU']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['AP']['NodeCount']) DS.resetDemand(demand) DS.setConfig(config) coef = (ret["ServerState"]["CostPerServer"] / ret["ServerState"]["ProfitConstant"]) DS.setCoef(coef) # f = open('myfile.txt','w') # f.write(str(ret)) # python will convert \n to os.linesep # f.close() conn.close() print 'Turn: ' + str(ret['ServerState']['TurnNo']) print "WEB capacity: " + str(capacity[0]) print "JAVA capacity: " + str(capacity[1]) print "DB capacity: " + str(capacity[2]) print "ServerCost: " + str(ret["ServerState"]["CostPerServer"]) #print didGrid #if didGrid: try: inf = str(ret["ServerState"]["InfraStructureUpgradeState"]["Value"]) if inf >=0: print "INFRA value: " + inf except: pass try: grid = str(ret["ServerState"]["ResearchUpgradeState"]["GRID"]) if grid != "-1": print "---Researching: "+ "GRID" +"---\nTurns Left: " + grid if int(grid) <= 1441 and int(grid) >= 1430: #infra = True pass else: infra = False else: print "GRID UPGRADE COMPLETE" except: pass try: green = str(ret["ServerState"]["ResearchUpgradeState"]["GREEN"]) if green != "-1": print "---Researching: "+ "GREEN" +"---\nTurns Left: " + green else: print "GREEN UPGRADE COMPLETE" except: pass print demand print ' ' + str(config[0]) + ' ' + str(config[1]) + ' ' + str(config[2]) + ' ' + '\n ' + str(config[3]) + ' ' + str(config[4]) + ' ' + str(config[5]) + ' ' + '\n ' + str(config[6]) + ' ' + str(config[7]) + ' ' + str(config[8]) print '' conn.close()
# resp = requests.get(_BASE_URL.format(21.82706108, 75.613033)).json()["boundingbox"] # box = resp[0] + "," + resp[2] + "," + resp[1] + "," + resp[3] # coords = str(box).split(',') # first = Location(float(coords[0]), float(coords[1])) # second = Location(float(coords[2]), float(coords[3])) first_lat = float(input("Enter first latitude:")) first_lon = float(input("Enter first longitude:")) second_lat = float(input("Enter second latitude:")) second_lon = float(input("Enter second longitude:")) first = Location(first_lat, first_lon) second = Location(second_lat, second_lon) name = input("Enter city name:") accuracy = int(input("Enter accuracy in meteres:")) locParser = LocationParser(first, second, accuracy=accuracy) locParser.parse() elevParser = ElevationParser(locParser.grid) DataStore.save(elevParser.grid, name=name) data = DataStore.load(name) s = getWatershed(data, 100) m = RiskMap(s, 10) m.generate_kml(name=name)
def __init__(self): tok = "8051bf89-e115-4147-8e5a-ff9d6f39f0d7" url = "hermes.wha.la" value = { "Command": "INIT", "Token": tok } headers = { 'Content-type': 'application/json', 'Accept': 'application/json', } jvalue = json.dumps(value) conn = httplib.HTTPConnection(url, 80) conn.request('POST', '/api/hermes', jvalue, headers) response = conn.getresponse() ret = json.loads(str((response.status, response.reason, response.read())[2])) conn.close() DS = DataStore(ret["ServerState"]["ServerTiers"]["DB"]["ServerStartTurnTime"]) ctrl = controller() # myFile = open('output.txt', 'w') # myFile.write(str(ret)) # myFile.close() looptime = ret['ServerState']['ServerTiers']['DB']['ServerStartTurnTime'] coef = (ret["ServerState"]["CostPerServer"] / ret["ServerState"]["ProfitConstant"]) WebRefresh = ret["ServerState"]["ServerTiers"]["WEB"]["ServerStartTurnTime"] JavaRefresh = ret["ServerState"]["ServerTiers"]["JAVA"]["ServerStartTurnTime"] DBRefresh = ret["ServerState"]["ServerTiers"]["DB"]["ServerStartTurnTime"] DS.setCoef(coef) infra = False p = None research = None didGrid = False progression = [None, "GRID", "GREEN", None ] #while ret['ServerState']['TurnNo'] < 10080: while True: x = 0 value = { "Command": "PLAY", "Token": tok } headers = { 'Content-type': 'application/json', 'Accept': 'application/json', } jvalue = json.dumps(value) conn = httplib.HTTPConnection(url, 80) conn.request('POST', '/api/hermes', jvalue, headers) response = conn.getresponse() ret = json.loads(str((response.status, response.reason, response.read())[2])) turnnumber = ret['ServerState']['TurnNo'] #Get demand from server demand = [ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['NA']['NoOfTransactionsInput']] demand.append(ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['EU']['NoOfTransactionsInput']) demand.append(ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['AP']['NoOfTransactionsInput']) config = [ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['NA']['NodeCount']] config.append(ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['EU']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['AP']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['JAVA']['ServerRegions']['NA']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['JAVA']['ServerRegions']['EU']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['JAVA']['ServerRegions']['AP']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['NA']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['EU']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['AP']['NodeCount']) # if turnnumber % (DBRefresh*2): # DS.resetDemand(demand) # else: # DS.avgDemand(demand) DS.runningDemand(demand) DS.setConfig(config) coef = (ret["ServerState"]["CostPerServer"] / ret["ServerState"]["ProfitConstant"]) DS.setCoef(coef) conn.close() lastProfit = ret["ServerState"]["ProfitEarned"] GridCost = ret["ServerState"]["ResearchUpgradeLevels"][1]["UpgradeCost"] GridTurns = ret["ServerState"]["ResearchUpgradeLevels"][1]["NoOfTurnsRequired"] GridTotalCost = GridTurns * GridCost #init research when its swaggin to do so if ret['ServerState']['TurnNo'] <= 2000 and ret["ServerState"]["ProfitAccumulated"] >= GridTotalCost/8 and GridCost < (lastProfit - (lastProfit/3)): didGrid = True try: if ret["ServerState"]["ResearchUpgradeState"]["GRID"] == -1: #research = "GREEN" pass except: research = "GRID" pass #p = research #Calculate free space #AVERAGE CAPACITY # capacity = [ ( ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2] # capacity.append(( ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2) # capacity.append(( ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2) #97% capacity = [ ( ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']+ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] ) / 3] capacity.append(( ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] ) / 3) capacity.append(( ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit']) / 3) #93% # capacity = [ ( ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']+ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 3] # capacity.append(( ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 3) # capacity.append(( ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']) / 3) #100% CAPACITY # capacity = [ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit']] # capacity.append(ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit']) # capacity.append(ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit']) # #90% CAPACITY # capacity = [ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']] # capacity.append(ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']) # capacity.append(ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']) DS.setCapacity(capacity) webchanges = [0,0,0,0,0,0,0,0,0] javachanges = [0,0,0,0,0,0,0,0,0] dbchanges = [0,0,0,0,0,0,0,0,0] if turnnumber%WebRefresh == 0: webchanges = ctrl.calcWeb(DS,WebRefresh*1.5) if turnnumber%JavaRefresh == 0: javachanges = ctrl.calcJava(DS,JavaRefresh*1.5 ) if turnnumber%DBRefresh == 0: dbchanges = ctrl.calcDB(DS,DBRefresh*1.5) #create 'changes' to know what servers to bring up/down changes = [] changes.append(webchanges[0]) changes.append(webchanges[1]) changes.append(webchanges[2]) changes.append(javachanges[3]) changes.append(javachanges[4]) changes.append(javachanges[5]) changes.append(dbchanges[6]) changes.append(dbchanges[7]) changes.append(dbchanges[8]) jsonchange = {"Servers":{ "WEB":{ "ServerRegions":{ "AP":{ "NodeCount":changes[2] }, "EU":{ "NodeCount":changes[1] }, "NA":{ "NodeCount":changes[0] } } }, "JAVA":{ "ServerRegions":{ "NA":{ "NodeCount":changes[3] }, "EU":{ "NodeCount":changes[4] }, "AP":{ "NodeCount":changes[5] } } }, "DB":{ "ServerRegions":{ "NA":{ "NodeCount":changes[6] }, "EU":{ "NodeCount":changes[7] }, "AP":{ "NodeCount":changes[8] } } } }, "UpgradeInfraStructure": infra, "UpgradeToResearch": research } if research != None: research = None value = { "Command": "CHNG", "Token": tok, "ChangeRequest": jsonchange } headers = { 'Content-type': 'application/json', 'Accept': 'application/json', } jvalue = json.dumps(value) conn = httplib.HTTPConnection(url, 80) conn.request('POST', '/api/hermes', jvalue, headers) response = conn.getresponse() ret2 = json.loads(str((response.status, response.reason, response.read())[2])) conn.close() #print Stuff print 'Turn: ' + str(ret['ServerState']['TurnNo']) print 'Total Profit: $' + str(ret["ServerState"]["ProfitAccumulated"]) # print "WEB capacity: " + str(capacity[0]) # print "JAVA capacity: " + str(capacity[1]) # print "DB capacity: " + str(capacity[2]) # print "ServerCost: " + str(ret["ServerState"]["CostPerServer"]) #print didGrid #if didGrid: try: inf = str(ret["ServerState"]["InfraStructureUpgradeState"]["Value"]) if inf >=0: #print "INFRA value: " + inf pass except: pass try: grid = str(ret["ServerState"]["ResearchUpgradeState"]["GRID"]) if grid != "-1": print "---Researching: "+ "GRID" +"---\nTurns Left: " + grid if int(grid) <= 1441 and int(grid) >= 1430: #infra = True pass else: infra = False else: print "GRID UPGRADE COMPLETE" except: pass try: green = str(ret["ServerState"]["ResearchUpgradeState"]["GREEN"]) if green != "-1": print "---Researching: "+ "GREEN" +"---\nTurns Left: " + green else: print "GREEN UPGRADE COMPLETE" except: pass print demand print ' ' + str(config[0]) + ' ' + str(config[1]) + ' ' + str(config[2]) + ' ' + '\n ' + str(config[3]) + ' ' + str(config[4]) + ' ' + str(config[5]) + ' ' + '\n ' + str(config[6]) + ' ' + str(config[7]) + ' ' + str(config[8]) print '' conn.close()
id=1234, gender="m", linkedin="https://www.linkedin.com/in/christian-bøgelund-92747799/", study_line="Cand. Alt", university="DTU", content=LOREM ) mentees.append(mentee) mentor = Mentor( name="En helt anden mentor, der ikke er Nana", age=0, id=2345, gender="m", linkedin="https://www.linkedin.com/in/christian-bøgelund-92747799/", study_line="Cand. Alt", company="Microsoft", university="", company_type="", position="CEO", content=LOREM ) mentors.append(mentor) """ # mentees, mentors = zip(*hungarian_main()) # show_matches(mentees, mentors) ds = DataStore()
""" # Temp import for testing import sys import os sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)) + '/..') from Report import Report from ReportConfig import ReportConfig from DataStore import DataStore import pandas as pd import unsorted import datetime print("Loading ENCV indicators into DataStore") ds = DataStore(['ENCV']) print("Querying KB.1-D") ds.query_indicators(program='ENCV', indicator='KB.1', level='D') config = ReportConfig() ''' print("Removing the NDA threshold") config.NDA_threshold = 0.0 ''' for row in ds.last_query.iterrows(): print("Processing course", row[1]["Course #"]) indicator_data, bins = unsorted.parse_row(row[1], config.header_attribs) indicator_data['Program'] = 'ENCV'
def __init__( self, config, programs=None, cohorts=None, whitelist=None, ds=None, indicators_loc=None, grades_loc=None, histograms_loc=None, ): """Object Initialization Args: config(ReportConfig): A ReportConfig object programs(string or list(string)): A list of programs to generate indicators for. Defaults to using all programs. If only one string is passed in, it will get put into a size-one list. cohorts(int or list(int)): A list of cohorts to pull data from. Defaults to using all cohorts. If only one cohort is passed in, it will get put into a size-one list. whitelist(dictionary): A dictionary of lists, keyed by the only stuff to parse (e.g. 'course', 'indicator', etc.) and filled with the specific values to uniquely parse. Defaults to no whitelist. The lists in the dictionaries can be partial (i.e. if you pass 'KB' as part of the 'indicator' whitelist, it will pull all 'KB' indicators). If the lists contain only one string, that string gets put into a size-one list. ds(DataStore): A DataStore object. Defaults to generating one based on the whitelist entries for 'programs' indicators_loc(string): The location of the indicator sheets. Defaults to searching an "Indicators" folder in the directory above the project (see ReportConfig for more info) grades_loc(string): The location of the grades sheets. Defaults to searching a "Grades" folder in the directory above the project (see ReportConfig for more info) histograms_loc(string): The location to store histograms. Defaults to using a "Histograms" folder in the directory above the project (see ReportConfig for more info) """ logging.info("Start of AutoGenerator initialization") self.config = config # logging.info("Initializing whitelist") self.whitelist = whitelist # Ensure that all whitelist entries are lists (if it exists, that is) if self.whitelist: logging.debug("Checking whitelist for list validity") for entry in self.whitelist.keys(): self.whitelist[entry] = _check_list(self.whitelist[entry]) # if type(self.whitelist[entry]) is not type(list()): # logging.debug("Whitelist entry %s is not a list. Converting to a one-size list", entry) # # Change to one-size list # self.whitelist[entry] = [self.whitelist[entry]] # logging.info("Initializing cohorts list") self.cohorts = cohorts # Ensure that all cohorts entries are lists (if it exists, that is) if self.cohorts: logging.debug("Checking cohorts variable for list validity") self.cohorts = _check_list(self.cohorts) # logging.info("Initializing program list") self.programs = programs # Ensure list validity of programs if self.programs: logging.debug("Checking programs for list validity") self.programs = _check_list(self.programs) # Use all programs if none were provided if not self.programs: logging.debug( "Programs not passed as paramater. Using list of all programs") self.programs = globals.all_programs # # Same check as whitelist - ensure that self.programs is a list # if type(self.programs) is not type(list()): # logging.debug("Programs was not passed in list format. Converting to a one-size list") # self.programs = [self.programs] # If any of the file location parameters were passed in, overwrite what ReportConfig has if indicators_loc: self.config.indicators_loc = indicators_loc if grades_loc: self.config.grades_loc = grades_loc if histograms_loc: self.config.histograms_loc = histograms_loc logging.debug("Indicators location is %s", self.config.indicators_loc) logging.debug("Grades location is %s", self.config.grades_loc) logging.debug("Histograms location is %s", self.config.histograms_loc) # Check to see if a DataStore was passed to init, create one if not if not ds: logging.debug( "No DataStore object was passed to init; creating one now") self.ds = DataStore(programs=self.programs, indicators_loc=self.config.indicators_loc, grades_loc=self.config.grades_loc) # Make sure that the histograms folder exists logging.info( "Setting up output directories (Missing Data & Histograms)") os.makedirs(os.path.dirname(__file__) + '/../Missing Data', exist_ok=True) os.makedirs(self.config.histograms_loc, exist_ok=True) logging.info("ReportGenerator initialization done!")
def __init__(self): #实例化同级类 self.parser = DataParser() self.datastore = DataStore()
def __init__(self): self.data_store = DataStore() self.url_manager = UrlManager(self.data_store) self.strategy_container = ParserStrategyContainer() self.downloader = HtmlDownloader()
class BikeService: """A class used to represent the business logic and delegates actions to data layer.""" def __init__(self): """Default Constructor to initialize data/persistence layer and fetch records from csv.""" self.data_store = DataStore() self.list_records = self.data_store.get_all_records() def print_all(self): """Method to print all the records to console and every 9 records my name - Adam Mohr.""" count = 0 for row in self.list_records: print(row) count += 1 if count % 9 == 0: self.print_adam() self.print_length() @staticmethod def get_input_new_record(): """Helper method to get input from user.""" year = input("Enter year: ") sex = input("Enter sex: ") age_group = input("Enter age group: ") student_response = input("Enter student response: ") vector = input("Enter vector: ") coordinate = input("Enter coordinate: ") value = input("Enter value: ") new_record = BikeRecord(year, "Canada", "", sex, age_group, student_response, "Percent", "239", "units", "0", vector, coordinate, value, "", "", "", "0") return new_record def insert_bike_record(self): """Method to insert new Bike Record at end of the list. Hardcoded certain values and left some fields blank to match formatting found in original csv file.""" self.list_records.append(self.get_input_new_record()) print("Record added.") @staticmethod def print_adam(): """Method to print the author's name - Adam Mohr.""" print("Program by: Adam Mohr") def print_length(self): """Helper method to print length of list i.e. how many records.""" print(len(self.list_records), "Records found") def save_records(self): """Method to save a new csv file to disk using DataStore method.""" self.data_store.save_records_to_disk() def update_record(self): """Helper method to get valid list index number from user.""" self.print_length() try: index = int( input("Enter index of record to update (starts at index 0): ")) self.list_records[index] = self.get_input_new_record() print("Record index", index, "updated.") except IndexError as e: print(e, "please give valid index number (starts at index 0).") except ValueError as e: print(e, "please give valid integer number.") def show_record(self): """Method to show a single record.""" self.print_length() try: index = int( input("Enter index of record to show (starts at index 0): ")) print(self.list_records[index]) except IndexError as e: print(e, "please give valid index number (starts at index 0).") except ValueError as e: print(e, "please give valid integer number.") def delete_record(self): """Method to delete a single record.""" self.print_length() try: index = int( input("Enter index of record to delete (starts at index 0): ")) self.list_records.pop(index) print("Record index", index, "deleted.") except IndexError as e: print(e, "please give valid index number (starts at index 0).") except ValueError as e: print(e, "please give valid integer number.") def sort_column(self): """Method to sort by column. Input is retrieved from user to determine which column to sort. Inline lambda function is used with a key that identifies the sorting property.""" column = input( "Enter column number to sort by: 0 = Sex, 1 = Age group, 2 = Student response, 3 = VECTOR, " "4 = COORDINATE, " " 5 = VALUE: ") if column == "0": self.list_records.sort(key=lambda x: x.sex) elif column == "1": self.list_records.sort(key=lambda x: x.age) elif column == "2": self.list_records.sort(key=lambda x: x.response) elif column == "3": self.list_records.sort(key=lambda x: x.vector) elif column == "4": self.list_records.sort(key=lambda x: x.coordinate) elif column == "5": self.list_records.sort(key=lambda x: int(x.value)) else: print("Not a valid column number") @staticmethod def print_stars(num): """Static helper method to print out the number of stars to use in the histogram.""" for i in range(num): print("*", end='') print() def print_chart(self, rarely, sometimes, often, always, dont): """Used with print_stars to create the formatted histogram and output it to console.""" print("How often do you wear your bicycle helmet while cycling?") print("Rarely or never (", end='') print(rarely, ") |", end='', sep='') self.print_stars(rarely) print("Sometimes (", end='') print(sometimes, ") |", end='', sep='') self.print_stars(sometimes) print("Often (", end='') print(often, ") |", end='', sep='') self.print_stars(often) print("Always (", end='') print(always, ") |", end='', sep='') self.print_stars(always) print("Do not ride a bicycle (", end='') print(dont, ") |", end='', sep='') self.print_stars(dont) def generate_histogram(self): """Retrieves input and generates histogram data based on user input. Loops through list of records and increments the specific count when the response condition is satisfied. Uses print_chart to output the data.""" print("How you would like to visualize data chart?") print( "a: age and gender, b: aggregate female, c: aggregate male, d: 11 years, e: 13 years, " "f: 15 years, g: all data") response = input("Please enter an option found above: ") response = response.lower() count_rarely = 0 count_sometimes = 0 count_often = 0 count_always = 0 count_dont = 0 if response == "a": gender = input("Male or female? Enter m: male, f: female: ") gender = gender.lower() if gender == "m": gender = "Males" elif gender == "f": gender = "Females" else: print("Please enter a valid option. Back to main menu.") return age = input("Age range? Enter 11, 13, or 15: ") if age == "11": age = "11 years" elif age == "13": age = "13 years" elif age == "15": age = "15 years" else: print("Please enter a valid option. Back to main menu.") return for record in self.list_records: if record.sex == gender and record.age == age and record.response == "Rarely or never": count_rarely += 1 elif record.sex == gender and record.age == age and record.response == "Sometimes": count_sometimes += 1 elif record.sex == gender and record.age == age and record.response == "Often": count_often += 1 elif record.sex == gender and record.age == age and record.response == "Always": count_always += 1 elif record.sex == gender and record.age == age and record.response == "Do not ride a bicycle": count_dont += 1 print(gender, ", ", age, sep='') self.print_chart(count_rarely, count_sometimes, count_often, count_always, count_dont) elif response == "b": for record in self.list_records: if record.sex == "Females" and record.response == "Rarely or never": count_rarely += 1 elif record.sex == "Females" and record.response == "Sometimes": count_sometimes += 1 elif record.sex == "Females" and record.response == "Often": count_often += 1 elif record.sex == "Females" and record.response == "Always": count_always += 1 elif record.sex == "Females" and record.response == "Do not ride a bicycle": count_dont += 1 print("Females, all ages:") self.print_chart(count_rarely, count_sometimes, count_often, count_always, count_dont) elif response == "c": for record in self.list_records: if record.sex == "Males" and record.response == "Rarely or never": count_rarely += 1 elif record.sex == "Males" and record.response == "Sometimes": count_sometimes += 1 elif record.sex == "Males" and record.response == "Often": count_often += 1 elif record.sex == "Males" and record.response == "Always": count_always += 1 elif record.sex == "Males" and record.response == "Do not ride a bicycle": count_dont += 1 print("Males, all ages:") self.print_chart(count_rarely, count_sometimes, count_often, count_always, count_dont) elif response == "d": for record in self.list_records: if record.age == "11 years" and record.response == "Rarely or never": count_rarely += 1 elif record.age == "11 years" and record.response == "Sometimes": count_sometimes += 1 elif record.age == "11 years" and record.response == "Often": count_often += 1 elif record.age == "11 years" and record.response == "Always": count_always += 1 elif record.age == "11 years" and record.response == "Do not ride a bicycle": count_dont += 1 print("11 years old:") self.print_chart(count_rarely, count_sometimes, count_often, count_always, count_dont) elif response == "e": for record in self.list_records: if record.age == "13 years" and record.response == "Rarely or never": count_rarely += 1 elif record.age == "13 years" and record.response == "Sometimes": count_sometimes += 1 elif record.age == "13 years" and record.response == "Often": count_often += 1 elif record.age == "13 years" and record.response == "Always": count_always += 1 elif record.age == "13 years" and record.response == "Do not ride a bicycle": count_dont += 1 print("13 years old:") self.print_chart(count_rarely, count_sometimes, count_often, count_always, count_dont) elif response == "f": for record in self.list_records: if record.age == "15 years" and record.response == "Rarely or never": count_rarely += 1 elif record.age == "15 years" and record.response == "Sometimes": count_sometimes += 1 elif record.age == "15 years" and record.response == "Often": count_often += 1 elif record.age == "15 years" and record.response == "Always": count_always += 1 elif record.age == "15 years" and record.response == "Do not ride a bicycle": count_dont += 1 print("15 years old:") self.print_chart(count_rarely, count_sometimes, count_often, count_always, count_dont) elif response == "g": for record in self.list_records: if record.response == "Rarely or never": count_rarely += 1 elif record.response == "Sometimes": count_sometimes += 1 elif record.response == "Often": count_often += 1 elif record.response == "Always": count_always += 1 elif record.response == "Do not ride a bicycle": count_dont += 1 print("Males, Females, All ages:") self.print_chart(count_rarely, count_sometimes, count_often, count_always, count_dont) else: print("Please enter a valid option. Back to main menu.")
class MatchRater: def __init__(self, str_id, df=None, rematch=False): self.df = df self.str_id = str_id self.initialize() self.ds = DataStore() if rematch: self.ds.rematch = True def initialize(self): self.root = Tk() self.root.title("MatchRater 5000") self.root.bind('<Return>', self.YES_onclick) self.root.bind('<BackSpace>', self.NO_onclick) self.root.bind('<space>', self.MAYBE_onclick) self.root.columnconfigure(0, pad=100, minsize=450) self.root.columnconfigure(1, pad=10, minsize=450) self.root.rowconfigure(0, minsize=50) def add_label(self, text, row, column, pady=10, padx=25, sticky=W): lbl = Label(self.root, text=text, wraplength=350) lbl.grid(row=row, column=column, pady=pady, padx=padx, sticky=sticky) def gui_test(self): self.add_label("Mentee name", 0, 0, padx=10, sticky=N) self.add_label("Position", 1, 0) self.add_label("University", 2, 0) self.add_label(LOREM, 3, 0) self.add_label("Suggestion 1", 4, 0) self.add_label("Suggestion 2", 5, 0) self.add_label("Suggestion 3", 6, 0) self.add_label("Mentee name", 0, 1, padx=10, sticky=N) self.add_label("Position", 1, 1) self.add_label("Company", 2, 1) self.add_label(LOREM, 3, 1) self.add_label("Suggestion 1", 4, 1) self.add_label("Suggestion 2", 5, 1) self.add_label("Suggestion 3", 6, 1) self.setup_bottom() def setup_bottom(self, match_score=""): lbl = Label( self.root, text=str(match_score) + "\nWas this match good/plausible? E.g. could it have happened when we were matching?" ) lbl.grid(row=7, column=0, columnspan=2, pady=(50, 5), sticky=S) btn = Button(self.root, text="Yes", command=self.YES_onclick) btn.grid(row=8, column=0, sticky=E, pady=(0, 50)) btn = Button(self.root, text="No", command=self.NO_onclick) btn.grid(row=8, column=1, sticky=W, pady=(0, 50)) def show_match(self, mentor, mentee, score): self.mentor = mentor self.mentee = mentee self.add_label(mentee.name, 0, 0, padx=10, sticky=N) self.add_label(mentee.position, 1, 0) self.add_label(mentee.university, 2, 0) self.add_label(mentee.content, 3, 0) """ self.add_label(mentee.priorities[0], 4, 0) self.add_label(mentee.priorities[1], 5, 0) self.add_label(mentee.priorities[2], 6, 0)""" self.add_label(mentor.name, 0, 1, padx=10, sticky=N) self.add_label(mentor.position, 1, 1) self.add_label(mentor.company, 2, 1) self.add_label(mentor.content, 3, 1) """self.add_label(mentor.priorities[0], 4, 1) self.add_label(mentor.priorities[1], 5, 1) self.add_label(mentor.priorities[2], 6, 1)""" self.setup_bottom(match_score=score) def start(self): self.root.mainloop() def NO_onclick(self, event=None): print("The match between " + self.mentee.name + " and " + self.mentor.name + " was bad") self.df = self.df.append( { "mentee_id": self.mentee.id, "mentor_id": self.mentor.id, "match_rating": 0 }, ignore_index=True) for child in self.root.winfo_children(): child.destroy() self.root.quit() # self.root.destroy() def MAYBE_onclick(self, event=None): print("The match between " + self.mentee.name + " and " + self.mentor.name + " was good!") self.df = self.df.append( { "mentee_id": self.mentee.id, "mentor_id": self.mentor.id, "match_rating": -1 }, ignore_index=True) for child in self.root.winfo_children(): child.destroy() self.root.quit() # self.root.destroy() def YES_onclick(self, event=None): print("The match between " + self.mentee.name + " and " + self.mentor.name + " was good!") self.df = self.df.append( { "mentee_id": self.mentee.id, "mentor_id": self.mentor.id, "match_rating": 1 }, ignore_index=True) for child in self.root.winfo_children(): child.destroy() self.root.quit() # self.root.destroy() # TODO: Fix issues with saving (probably caused by issue in YES/NO_onclick) def export_csv(self): self.ds.save_to_disk(df=self.df) self.ds.append_matching_info(df=self.df, str_id=self.str_id) def kill(self): self.root.destroy() def get_cache(self): cached_matches = {} duplicated = {} for file in os.listdir("data/match_rating"): results = self.ds.load_rating_file(file) for i, result in results.iterrows(): dict_key = str(result["mentor_id"]) + str(result["mentee_id"]) # If the key already exists if dict_key in cached_matches and dict_key not in duplicated: if result["match_rating"] != cached_matches[dict_key]: duplicated[dict_key] = result["match_rating"] del (cached_matches[dict_key]) continue # If the key does not exist cached_matches[dict_key] = result["match_rating"] return cached_matches def rate_match(self, mentor, mentee, rating): self.df = self.df.append( { "mentee_id": mentee.id, "mentor_id": mentor.id, "match_rating": rating }, ignore_index=True)
def __init__(self): tok = "8051bf89-e115-4147-8e5a-ff9d6f39f0d7" url = "hermes.wha.la" value = {"Command": "INIT", "Token": tok} headers = { 'Content-type': 'application/json', 'Accept': 'application/json', } jvalue = json.dumps(value) conn = httplib.HTTPConnection(url, 80) conn.request('POST', '/api/hermes', jvalue, headers) response = conn.getresponse() ret = json.loads( str((response.status, response.reason, response.read())[2])) conn.close() DS = DataStore( ret["ServerState"]["ServerTiers"]["DB"]["ServerStartTurnTime"]) ctrl = controller() # myFile = open('output.txt', 'w') # myFile.write(str(ret)) # myFile.close() looptime = ret['ServerState']['ServerTiers']['DB'][ 'ServerStartTurnTime'] coef = (ret["ServerState"]["CostPerServer"] / ret["ServerState"]["ProfitConstant"]) WebRefresh = ret["ServerState"]["ServerTiers"]["WEB"][ "ServerStartTurnTime"] JavaRefresh = ret["ServerState"]["ServerTiers"]["JAVA"][ "ServerStartTurnTime"] DBRefresh = ret["ServerState"]["ServerTiers"]["DB"][ "ServerStartTurnTime"] DS.setCoef(coef) infra = False p = None research = None didGrid = False progression = [None, "GRID", "GREEN", None] #while ret['ServerState']['TurnNo'] < 10080: while True: x = 0 value = {"Command": "PLAY", "Token": tok} headers = { 'Content-type': 'application/json', 'Accept': 'application/json', } jvalue = json.dumps(value) conn = httplib.HTTPConnection(url, 80) conn.request('POST', '/api/hermes', jvalue, headers) response = conn.getresponse() ret = json.loads( str((response.status, response.reason, response.read())[2])) turnnumber = ret['ServerState']['TurnNo'] #Get demand from server demand = [ ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['NA'] ['NoOfTransactionsInput'] ] demand.append(ret['ServerState']['ServerTiers']['WEB'] ['ServerRegions']['EU']['NoOfTransactionsInput']) demand.append(ret['ServerState']['ServerTiers']['WEB'] ['ServerRegions']['AP']['NoOfTransactionsInput']) config = [ ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['NA'] ['NodeCount'] ] config.append(ret['ServerState']['ServerTiers']['WEB'] ['ServerRegions']['EU']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['WEB'] ['ServerRegions']['AP']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['JAVA'] ['ServerRegions']['NA']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['JAVA'] ['ServerRegions']['EU']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['JAVA'] ['ServerRegions']['AP']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['DB'] ['ServerRegions']['NA']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['DB'] ['ServerRegions']['EU']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['DB'] ['ServerRegions']['AP']['NodeCount']) # if turnnumber % (DBRefresh*2): # DS.resetDemand(demand) # else: # DS.avgDemand(demand) DS.runningDemand(demand) DS.setConfig(config) coef = (ret["ServerState"]["CostPerServer"] / ret["ServerState"]["ProfitConstant"]) DS.setCoef(coef) conn.close() lastProfit = ret["ServerState"]["ProfitEarned"] GridCost = ret["ServerState"]["ResearchUpgradeLevels"][1][ "UpgradeCost"] GridTurns = ret["ServerState"]["ResearchUpgradeLevels"][1][ "NoOfTurnsRequired"] GridTotalCost = GridTurns * GridCost #init research when its swaggin to do so if ret['ServerState']['TurnNo'] <= 2000 and ret["ServerState"][ "ProfitAccumulated"] >= GridTotalCost / 8 and GridCost < ( lastProfit - (lastProfit / 3)): didGrid = True try: if ret["ServerState"]["ResearchUpgradeState"][ "GRID"] == -1: #research = "GREEN" pass except: research = "GRID" pass #p = research #Calculate free space #AVERAGE CAPACITY # capacity = [ ( ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2] # capacity.append(( ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2) # capacity.append(( ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2) #97% capacity = [ (ret['ServerState']['ServerTiers']['WEB']['ServerPerformance'] ['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance'] ['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance'] ['CapactityLevels'][0]['UpperLimit']) / 3 ] capacity.append( (ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance'] ['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance'] ['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance'] ['CapactityLevels'][0]['UpperLimit']) / 3) capacity.append( (ret['ServerState']['ServerTiers']['DB']['ServerPerformance'] ['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance'] ['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance'] ['CapactityLevels'][0]['UpperLimit']) / 3) #93% # capacity = [ ( ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']+ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 3] # capacity.append(( ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 3) # capacity.append(( ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']) / 3) #100% CAPACITY # capacity = [ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit']] # capacity.append(ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit']) # capacity.append(ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit']) # #90% CAPACITY # capacity = [ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']] # capacity.append(ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']) # capacity.append(ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']) DS.setCapacity(capacity) webchanges = [0, 0, 0, 0, 0, 0, 0, 0, 0] javachanges = [0, 0, 0, 0, 0, 0, 0, 0, 0] dbchanges = [0, 0, 0, 0, 0, 0, 0, 0, 0] if turnnumber % WebRefresh == 0: webchanges = ctrl.calcWeb(DS, WebRefresh * 1.5) if turnnumber % JavaRefresh == 0: javachanges = ctrl.calcJava(DS, JavaRefresh * 1.5) if turnnumber % DBRefresh == 0: dbchanges = ctrl.calcDB(DS, DBRefresh * 1.5) #create 'changes' to know what servers to bring up/down changes = [] changes.append(webchanges[0]) changes.append(webchanges[1]) changes.append(webchanges[2]) changes.append(javachanges[3]) changes.append(javachanges[4]) changes.append(javachanges[5]) changes.append(dbchanges[6]) changes.append(dbchanges[7]) changes.append(dbchanges[8]) jsonchange = { "Servers": { "WEB": { "ServerRegions": { "AP": { "NodeCount": changes[2] }, "EU": { "NodeCount": changes[1] }, "NA": { "NodeCount": changes[0] } } }, "JAVA": { "ServerRegions": { "NA": { "NodeCount": changes[3] }, "EU": { "NodeCount": changes[4] }, "AP": { "NodeCount": changes[5] } } }, "DB": { "ServerRegions": { "NA": { "NodeCount": changes[6] }, "EU": { "NodeCount": changes[7] }, "AP": { "NodeCount": changes[8] } } } }, "UpgradeInfraStructure": infra, "UpgradeToResearch": research } if research != None: research = None value = { "Command": "CHNG", "Token": tok, "ChangeRequest": jsonchange } headers = { 'Content-type': 'application/json', 'Accept': 'application/json', } jvalue = json.dumps(value) conn = httplib.HTTPConnection(url, 80) conn.request('POST', '/api/hermes', jvalue, headers) response = conn.getresponse() ret2 = json.loads( str((response.status, response.reason, response.read())[2])) conn.close() #print Stuff print 'Turn: ' + str(ret['ServerState']['TurnNo']) print 'Total Profit: $' + str( ret["ServerState"]["ProfitAccumulated"]) # print "WEB capacity: " + str(capacity[0]) # print "JAVA capacity: " + str(capacity[1]) # print "DB capacity: " + str(capacity[2]) # print "ServerCost: " + str(ret["ServerState"]["CostPerServer"]) #print didGrid #if didGrid: try: inf = str( ret["ServerState"]["InfraStructureUpgradeState"]["Value"]) if inf >= 0: #print "INFRA value: " + inf pass except: pass try: grid = str(ret["ServerState"]["ResearchUpgradeState"]["GRID"]) if grid != "-1": print "---Researching: " + "GRID" + "---\nTurns Left: " + grid if int(grid) <= 1441 and int(grid) >= 1430: #infra = True pass else: infra = False else: print "GRID UPGRADE COMPLETE" except: pass try: green = str( ret["ServerState"]["ResearchUpgradeState"]["GREEN"]) if green != "-1": print "---Researching: " + "GREEN" + "---\nTurns Left: " + green else: print "GREEN UPGRADE COMPLETE" except: pass print demand print ' ' + str(config[0]) + ' ' + str( config[1]) + ' ' + str(config[2]) + ' ' + '\n ' + str( config[3]) + ' ' + str(config[4]) + ' ' + str( config[5]) + ' ' + '\n ' + str( config[6]) + ' ' + str( config[7]) + ' ' + str(config[8]) print '' conn.close()
''' import logging #logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger() from Probe import Probe from PPPParam import PPPParam from ResGraph import ResGraph from ProbeController import ProbeController from DataStore import DataStore # ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- data_store = DataStore() probe1 = Probe('ping', "ping -c 1 localhost") probe2 = Probe('curl', "curl -s http://localhost/") probes = [probe1, probe2] ppp_params = [PPPParam(60), PPPParam(900), PPPParam()] for probe in probes: data_store.add(probe) data_store.commit() res_graph = ResGraph(probes, ppp_params) probe_controller = ProbeController(probes, res_graph, data_store) probe_controller.start()
SC_list = SeparateCollects(path_to_directory_of_ast_files, AGO.file_name[0:12]) print('Separated collect files for two collects: ', SC_list.collect_files) # Example storing frame stacks, labels, and mean object # centroids in a .hdf5 file. DataStore compresses the # .hdf5 file to reduce the required disk space. name_of_labels_csv =\ 'dataset_labels.csv' path_to_directory_of_fits_files =\ '/home/jermws/Sample Raven Images/FITS' name_of_hdf5_file_for_storage =\ 'labeled_datasets.hdf5' path_to_directory_for_storage =\ '/home/jermws/PycharmProjects/Machine_Learning_Updated/' DS = DataStore(name_of_labels_csv, path_to_directory_of_fits_files, path_to_directory_of_ast_files, name_of_hdf5_file_for_storage, path_to_directory_for_storage) # Save the list of data set names in the .hdf5 file # so they can be recalled when it comes time to read # the data back in. with open('dataset_names.csv', 'w') as f: w = csv.writer(f) for row in DS.dataset_names: w.writerow([row]) # Read in the data set names so they can be used to # label the chip stacks.
def __init__(self): """Default Constructor to initialize data/persistence layer and fetch records from csv.""" self.data_store = DataStore() self.list_records = self.data_store.get_all_records()
class SpiderMan: def __init__(self): self.data_store = DataStore() self.url_manager = UrlManager(self.data_store) self.strategy_container = ParserStrategyContainer() self.downloader = HtmlDownloader() def start(self): self.register_station() while self.has_task(): try: self.handle_station() self.handle_content() except Exception as e: print(e) def has_task(self): station_count = self.url_manager.get_stations_size() contents_count = self.url_manager.get_contents_size() all_count = station_count + contents_count return all_count > 0 def register_station(self): strategys = self.strategy_container.get_strategys() for key, value in strategys.items(): self.data_store.add_station_url(key, value.get_url()) def handle_station(self): data = self.url_manager.get_station_url() if data is None: return strategy = self.strategy_container.get_srategy(data['station']) # get all urls html = self.downloader.download(data['url']) #for test #fin = open("test", "w", encoding="utf-8") #fin.write(html) #fin = open("test", "r", encoding="utf-8") #html = fin.read(); content_urls = strategy.parser_station(data['url'], html) # store urls to mysql perform_count = 0 for content_url in content_urls: disease = content_url['disease'] tmp_url = content_url['url'] perform_count += self.url_manager.add_content_url( tmp_url, disease, data['station']) #updata sation state state = 1 if len(content_urls) > perform_count or len(content_urls) == 0: state = -1 self.data_store.updata_staion_state(state, data['station']) def handle_content(self): data = self.url_manager.get_content_url() strategy = self.strategy_container.get_srategy(data['station']) # get page contents html = self.downloader.download(data['url']) contents = strategy.parser_content(html) # store contents to mysql state = 0 code = 1 contents['knowledge_id'] = data['knowledge_id'] if self.data_store.add_content(contents) < 0: state = -1 #update state self.data_store.update_page_state(code, data['knowledge_id'])
def __init__(self): value = { "Command": "INIT", "Token": "8051bf89-e115-4147-8e5a-ff9d6f39f0d7" } headers = { 'Content-type': 'application/json', 'Accept': 'application/json', } jvalue = json.dumps(value) conn = httplib.HTTPConnection('107.20.243.77', 80) conn.request('POST', '/api/hermes', jvalue, headers) response = conn.getresponse() ret = json.loads( str((response.status, response.reason, response.read())[2])) conn.close() DS = DataStore() ctrl = controller() myFile = open('output.txt', 'w') myFile.write(str(ret)) myFile.close() looptime = ret['ServerState']['ServerTiers']['DB'][ 'ServerStartTurnTime'] coef = (ret["ServerState"]["CostPerServer"] / ret["ServerState"]["ProfitConstant"]) DS.setCoef(coef) infra = False p = None research = None didGrid = False progression = [None, "GRID", "GREEN", None] #while ret['ServerState']['TurnNo'] < 10080: while True: x = 0 while x <= looptime - 1: value = { "Command": "PLAY", "Token": "8051bf89-e115-4147-8e5a-ff9d6f39f0d7" } headers = { 'Content-type': 'application/json', 'Accept': 'application/json', } jvalue = json.dumps(value) conn = httplib.HTTPConnection('107.20.243.77', 80) conn.request('POST', '/api/hermes', jvalue, headers) response = conn.getresponse() ret = json.loads( str((response.status, response.reason, response.read())[2])) #Get demand from server demand = [ ret['ServerState']['ServerTiers']['DB']['ServerRegions'] ['NA']['NoOfTransactionsInput'] ] demand.append(ret['ServerState']['ServerTiers']['DB'] ['ServerRegions']['EU']['NoOfTransactionsInput']) demand.append(ret['ServerState']['ServerTiers']['DB'] ['ServerRegions']['AP']['NoOfTransactionsInput']) config = [ ret['ServerState']['ServerTiers']['WEB']['ServerRegions'] ['NA']['NodeCount'] ] config.append(ret['ServerState']['ServerTiers']['WEB'] ['ServerRegions']['EU']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['WEB'] ['ServerRegions']['AP']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['JAVA'] ['ServerRegions']['NA']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['JAVA'] ['ServerRegions']['EU']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['JAVA'] ['ServerRegions']['AP']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['DB'] ['ServerRegions']['NA']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['DB'] ['ServerRegions']['EU']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['DB'] ['ServerRegions']['AP']['NodeCount']) DS.avgDemand(demand) DS.setConfig(config) conn.close() x += 1 lastProfit = ret["ServerState"]["ProfitEarned"] GridCost = ret["ServerState"]["ResearchUpgradeLevels"][1][ "UpgradeCost"] GridTurns = ret["ServerState"]["ResearchUpgradeLevels"][1][ "NoOfTurnsRequired"] GridTotalCost = GridTurns * GridCost if ret['ServerState']['TurnNo'] <= 9000 and ret["ServerState"][ "ProfitAccumulated"] >= GridTotalCost / 10 and GridCost < ( lastProfit - (lastProfit / 3)): didGrid = True try: if ret["ServerState"]["ResearchUpgradeState"][ "GRID"] == -1: #research = "GREEN" pass except: research = "GRID" #p = research #Calculate free space #AVERAGE CAPACITY # capacity = [ ( ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2] # capacity.append(( ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2) # capacity.append(( ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2) #97% capacity = [ (ret['ServerState']['ServerTiers']['WEB']['ServerPerformance'] ['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance'] ['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance'] ['CapactityLevels'][0]['UpperLimit']) / 3 ] capacity.append( (ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance'] ['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance'] ['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance'] ['CapactityLevels'][0]['UpperLimit']) / 3) capacity.append( (ret['ServerState']['ServerTiers']['DB']['ServerPerformance'] ['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance'] ['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance'] ['CapactityLevels'][0]['UpperLimit']) / 3) #93% # capacity = [ ( ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']+ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 3] # capacity.append(( ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 3) # capacity.append(( ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']) / 3) #100% CAPACITY # capacity = [ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit']] # capacity.append(ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit']) # capacity.append(ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit']) # #90% CAPACITY # capacity = [ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']] # capacity.append(ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']) # capacity.append(ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']) DS.setCapacity(capacity) changes = ctrl.calc(DS) jsonchange = { "Servers": { "WEB": { "ServerRegions": { "AP": { "NodeCount": changes[2] }, "EU": { "NodeCount": changes[1] }, "NA": { "NodeCount": changes[0] } } }, "JAVA": { "ServerRegions": { "NA": { "NodeCount": changes[3] }, "EU": { "NodeCount": changes[4] }, "AP": { "NodeCount": changes[5] } } }, "DB": { "ServerRegions": { "NA": { "NodeCount": changes[6] }, "EU": { "NodeCount": changes[7] }, "AP": { "NodeCount": changes[8] } } } }, "UpgradeInfraStructure": infra, "UpgradeToResearch": research } if research != None: research = None value = { "Command": "CHNG", "Token": "8051bf89-e115-4147-8e5a-ff9d6f39f0d7", "ChangeRequest": jsonchange } headers = { 'Content-type': 'application/json', 'Accept': 'application/json', } jvalue = json.dumps(value) conn = httplib.HTTPConnection('107.20.243.77', 80) conn.request('POST', '/api/hermes', jvalue, headers) response = conn.getresponse() ret = json.loads( str((response.status, response.reason, response.read())[2])) # if research != None: # research = None # jsonchange = {"Servers":{ # "WEB":{"UpgradeToResearch": "Green"}}} # value = { # "Command": "CHNG", # "Token": "8051bf89-e115-4147-8e5a-ff9d6f39f0d7", # #"Token": "7440b0b0-c5a2-4ab3-bdc3-8935865bb9d1", # "ChangeRequest": jsonchange # } # headers = {'Content-type': 'application/json','Accept': 'application/json',} # jvalue = json.dumps(value) # conn = httplib.HTTPConnection('107.20.243.77', 80) # #conn = httplib.HTTPConnection('uat.hermes.wha.la', 80) # conn.request('POST', '/api/hermes', jvalue, headers) # response = conn.getresponse() # ret = json.loads(str((response.status, response.reason, response.read())[2])) #play value = { "Command": "PLAY", "Token": "8051bf89-e115-4147-8e5a-ff9d6f39f0d7" } headers = { 'Content-type': 'application/json', 'Accept': 'application/json', } jvalue = json.dumps(value) conn = httplib.HTTPConnection('107.20.243.77', 80) conn.request('POST', '/api/hermes', jvalue, headers) response = conn.getresponse() ret = json.loads( str((response.status, response.reason, response.read())[2])) #Get demand from server demand = [ ret['ServerState']['ServerTiers']['DB']['ServerRegions']['NA'] ['NoOfTransactionsInput'] ] demand.append(ret['ServerState']['ServerTiers']['DB'] ['ServerRegions']['EU']['NoOfTransactionsInput']) demand.append(ret['ServerState']['ServerTiers']['DB'] ['ServerRegions']['AP']['NoOfTransactionsInput']) config = [ ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['NA'] ['NodeCount'] ] config.append(ret['ServerState']['ServerTiers']['WEB'] ['ServerRegions']['EU']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['WEB'] ['ServerRegions']['AP']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['JAVA'] ['ServerRegions']['NA']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['JAVA'] ['ServerRegions']['EU']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['JAVA'] ['ServerRegions']['AP']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['DB'] ['ServerRegions']['NA']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['DB'] ['ServerRegions']['EU']['NodeCount']) config.append(ret['ServerState']['ServerTiers']['DB'] ['ServerRegions']['AP']['NodeCount']) DS.resetDemand(demand) DS.setConfig(config) coef = (ret["ServerState"]["CostPerServer"] / ret["ServerState"]["ProfitConstant"]) DS.setCoef(coef) # f = open('myfile.txt','w') # f.write(str(ret)) # python will convert \n to os.linesep # f.close() conn.close() print 'Turn: ' + str(ret['ServerState']['TurnNo']) print "WEB capacity: " + str(capacity[0]) print "JAVA capacity: " + str(capacity[1]) print "DB capacity: " + str(capacity[2]) print "ServerCost: " + str(ret["ServerState"]["CostPerServer"]) #print didGrid #if didGrid: try: inf = str( ret["ServerState"]["InfraStructureUpgradeState"]["Value"]) if inf >= 0: print "INFRA value: " + inf except: pass try: grid = str(ret["ServerState"]["ResearchUpgradeState"]["GRID"]) if grid != "-1": print "---Researching: " + "GRID" + "---\nTurns Left: " + grid if int(grid) <= 1441 and int(grid) >= 1430: #infra = True pass else: infra = False else: print "GRID UPGRADE COMPLETE" except: pass try: green = str( ret["ServerState"]["ResearchUpgradeState"]["GREEN"]) if green != "-1": print "---Researching: " + "GREEN" + "---\nTurns Left: " + green else: print "GREEN UPGRADE COMPLETE" except: pass print demand print ' ' + str(config[0]) + ' ' + str( config[1]) + ' ' + str(config[2]) + ' ' + '\n ' + str( config[3]) + ' ' + str(config[4]) + ' ' + str( config[5]) + ' ' + '\n ' + str( config[6]) + ' ' + str( config[7]) + ' ' + str(config[8]) print '' conn.close()
import pandas as pd import numpy as np from datetime import datetime import _pickle as pk import json from DataStore import DataStore # from FHMM import FHMM # from HMM import HMM, HMM_MAD from Preprocessing import Appliance, train_test_split, create_matrix #with open('aws_keys.json') as f: # data = json.load(f) # ACCESS_KEY = data['access-key'] # SECRET_ACCESS_KEY = data['secret-access-key'] DStore = DataStore('ngalvbucket1', 'house_1') all_channels = [1, 12, 5, 3, 10, 6, 9, 43, 7, 8] select_channels = [12, 5, 3, 10, 6, 9, 43, 8] # select_channels = [12, 5, 6] DStore.create_store(all_channels) #top_10 = DStore.select_top_k(10,'2013-08-01','2013-09-01') combined = DStore.create_combined_df('2013-06-01 00:00:00', '2013-10-31 23:59:59', select_channels=select_channels, freq='1Min') with open('combined.pkl', 'wb') as f: pk.dump(combined, f)
class ReportGenerator(object): """Assists with generating histogram reports Bundles all of the Report object generation things in one place and adds convenience methods for generation Attributes: config(ReportConfig): A ReportConfig object that the generator passes to each generated Report programs(list(string)): A list of programs (e.g. ENCM) to process. whitelist(dictionary): A dictionary including lists of the only things that should be processed. Options can include, but are not limited to, lists of: * Courses * Indicators * Assessments As long as the whitelist key matches closely to the column name in the indicator lookup table, it should be fine ds(DataStore): A DataStore object TODO: * Implement a blacklisting feature """ def __init__( self, config, programs=None, cohorts=None, whitelist=None, ds=None, indicators_loc=None, grades_loc=None, histograms_loc=None, ): """Object Initialization Args: config(ReportConfig): A ReportConfig object programs(string or list(string)): A list of programs to generate indicators for. Defaults to using all programs. If only one string is passed in, it will get put into a size-one list. cohorts(int or list(int)): A list of cohorts to pull data from. Defaults to using all cohorts. If only one cohort is passed in, it will get put into a size-one list. whitelist(dictionary): A dictionary of lists, keyed by the only stuff to parse (e.g. 'course', 'indicator', etc.) and filled with the specific values to uniquely parse. Defaults to no whitelist. The lists in the dictionaries can be partial (i.e. if you pass 'KB' as part of the 'indicator' whitelist, it will pull all 'KB' indicators). If the lists contain only one string, that string gets put into a size-one list. ds(DataStore): A DataStore object. Defaults to generating one based on the whitelist entries for 'programs' indicators_loc(string): The location of the indicator sheets. Defaults to searching an "Indicators" folder in the directory above the project (see ReportConfig for more info) grades_loc(string): The location of the grades sheets. Defaults to searching a "Grades" folder in the directory above the project (see ReportConfig for more info) histograms_loc(string): The location to store histograms. Defaults to using a "Histograms" folder in the directory above the project (see ReportConfig for more info) """ logging.info("Start of AutoGenerator initialization") self.config = config # logging.info("Initializing whitelist") self.whitelist = whitelist # Ensure that all whitelist entries are lists (if it exists, that is) if self.whitelist: logging.debug("Checking whitelist for list validity") for entry in self.whitelist.keys(): self.whitelist[entry] = _check_list(self.whitelist[entry]) # if type(self.whitelist[entry]) is not type(list()): # logging.debug("Whitelist entry %s is not a list. Converting to a one-size list", entry) # # Change to one-size list # self.whitelist[entry] = [self.whitelist[entry]] # logging.info("Initializing cohorts list") self.cohorts = cohorts # Ensure that all cohorts entries are lists (if it exists, that is) if self.cohorts: logging.debug("Checking cohorts variable for list validity") self.cohorts = _check_list(self.cohorts) # logging.info("Initializing program list") self.programs = programs # Ensure list validity of programs if self.programs: logging.debug("Checking programs for list validity") self.programs = _check_list(self.programs) # Use all programs if none were provided if not self.programs: logging.debug( "Programs not passed as paramater. Using list of all programs") self.programs = globals.all_programs # # Same check as whitelist - ensure that self.programs is a list # if type(self.programs) is not type(list()): # logging.debug("Programs was not passed in list format. Converting to a one-size list") # self.programs = [self.programs] # If any of the file location parameters were passed in, overwrite what ReportConfig has if indicators_loc: self.config.indicators_loc = indicators_loc if grades_loc: self.config.grades_loc = grades_loc if histograms_loc: self.config.histograms_loc = histograms_loc logging.debug("Indicators location is %s", self.config.indicators_loc) logging.debug("Grades location is %s", self.config.grades_loc) logging.debug("Histograms location is %s", self.config.histograms_loc) # Check to see if a DataStore was passed to init, create one if not if not ds: logging.debug( "No DataStore object was passed to init; creating one now") self.ds = DataStore(programs=self.programs, indicators_loc=self.config.indicators_loc, grades_loc=self.config.grades_loc) # Make sure that the histograms folder exists logging.info( "Setting up output directories (Missing Data & Histograms)") os.makedirs(os.path.dirname(__file__) + '/../Missing Data', exist_ok=True) os.makedirs(self.config.histograms_loc, exist_ok=True) logging.info("ReportGenerator initialization done!") def _parse_row(self, row, program): """Turn a row of a Pandas DataFrame into a dictionary and bin list The data stored in the master indicator spreadsheets needs to be cleaned up properly for a Report object to use it. This function does that based on things in the ReportConfig. Program gets passed to the function because that is currently not stored in the indicators spreadsheet and needs to end up on the Report object somehow. This seemed like the most reasonable place to do it. ReportConfig.header_attribs determines what data gets pulled from the spreadsheet. For a single entry in header_attribs, multiple values could get pulled and concatenated from the spreadsheet row. For example, writing just 'Course' in the header_attribs would join the columns 'Course #' and 'Course Description' using a ' - ' character Args: row(pd.Series): The row of a Pandas DataFrame (so a Pandas Series) to pull data from program(string): The program that the indicator file row belongs to Returns: dict: A dictionary containing the indicator information, keyed using instructions list(float): The bin ranges that have been converted from a comma-separated string to a list of floats. 0 gets appended to the front for NumPy histogram None, None: Returns 2 None values if non-number bins are found in the row """ # logging.info("Parsing a row from a Pandas DataFrame") # Handle the bins first since those are easy try: bins = [float(x) for x in row['Bins'].split(',')] except Exception: logging.warning( "ERROR: Non-number bins encountered in a lookup table") return None, None logging.debug("Bins parsed as:\t%s", ', '.join(str(x) for x in bins)) # Convert the comma-separated string into dictionary keys # logging.info("Creating a dictionary of indicator information") indicator_dict = { i: "" for i in [x.strip() for x in self.config.header_attribs.split(',')] } # logging.info("Filling the dictionary with information from the row") for key in indicator_dict.keys(): # Look for all occurrences where the key is found in the row's columns # and store that information in a list occurrences = [str(row[i]) for i in row.index if key in i] # Glue all collected data together with ' - ' characters # logging.debug("Indicator entry %s being filled with info from lookup table columns %s", # key, # ', '.join(occurrences) # ) indicator_dict[key] = ' - '.join(occurrences) # logging.debug("Entry [%s]: %s", key, indicator_dict[key]) # If Program is in indicator_dict, add the program parameter to indicator_dict if 'Program' in indicator_dict.keys(): indicator_dict['Program'] = program # logging.info("Returning indicator dictionary and bins from this row") return indicator_dict, bins def start_autogenerate(self): """Begin autogeneration of reports The general procedure goes as follows: * Iterate across the stored indicator lookup tables: * Query the indicator table using the whitelist * Iterate through the query: * Get indicator data from the row * Search for grades for the assessment tool * Iterate through each file found and save a histogram TODO: * Decouple the method * Change the histogram file naming conventions to be shorter * Allow customization of the file save name through ReportConfig """ logging.info("Beginning report autogeneration") logging.debug("Autogenerator set up to use programs %s", ', '.join(self.programs)) # Iterate across the list of programs for program in self.programs: logging.info("Generating reports for program %s", program) # logging.info("Getting a query list from the program's indicator lookup table") # Query the indicators DataFrame query = self.ds.query_indicators(program=program, dict_of_queries=self.whitelist) # Set up a file to store missing data in # logging.info("Starting a file to save missing data") missing_data = open( "../Missing Data/{} missing data.txt".format(program), "w+") # Iterate across each indicator (each row of the query) # logging.info("Beginning row iteration...") for rownumber, row in query.iterrows(): # Skip this row if the "Assessed" column is set to any form of "No" (also check if it's there) if 'Assessed' in row and row['Assessed'].lower() == 'no': # print("Found a non-assessed indicator") continue # Skip this row if no bins are defined if row['Bins'] in [np.nan, None]: logging.warning( "No bins (and likely no data) found for {} {} {} {}, skipping row" .format(row['Indicator #'], row['Level'], row['Course #'], row['Method of Assessment'])) logging.warning("Missing binning stored in separate file") # Log the missing data entry missing_data.write( "Missing bin ranges for {a} {b} ({c}-{d})\n".format( a=row["Course #"], b=row["Method of Assessment"], c=row['Indicator #'], d=row['Level'])) continue else: logging.debug("Processing data for {} {} {} {}".format( row['Indicator #'], row['Level'], row['Course #'], row['Method of Assessment'])) #------------------------------------------------------------------------------ # Obtain the necessary indicator data and bins from the row. header_attribs in # ReportConfig determine what gets pulled and what doesn't. These are also # JSON-loadable. Program gets added to indicator data later if required. #------------------------------------------------------------------------------ indicator_data, bins = self._parse_row(row, program) # If no bins were parsed, a histogram cannot be generated. Skip this indicator/row if not bins: logging.warning( "ERROR: No useable bins for {} {} {} {}, skipping row". format(row['Indicator #'], row['Level'], row['Course #'], row['Method of Assessment'])) # Log the missing bins missing_data.write( "No useable bins for {a} {b} ({c}-{d})\n".format( a=row["Course #"], b=row["Method of Assessment"], c=row['Indicator #'], d=row['Level'])) continue logging.debug("Indicator data obtained from the row: %s", str(indicator_data)) #----------------------------------------------------------------------------- # Search for grade files. Store all occurrences of the assessment data so # that a histogram can be generated for each data set. # # Generates a defaultdict like the following example: # { # "ENCM": [], # "Core": ["ENGI 1040 Circuits Grade - Core - Custom column names.xlsx"], # "ECE": ["ENGI 1040 Circuits Grade - ECE - Custom column names.xlsx"] # } # These are stored in lists because the original intention was to allow # multiple assessment files for the same indicator. #----------------------------------------------------------------------------- # Use the config parameter to get the backup subdirectories search_list = [program] + [ x.strip() for x in self.config.grade_backup_dirs.split(',') ] logging.debug("Beginning search for grade files. Priority: %s", ', '.join(search_list)) found_grade_files = grades_org.directory_search( course=row['Course #'], assessment=row['Method of Assessment'], main_dir=self.config.grades_loc, subdirs=search_list) # # Check the unique course table to see if the course has a unique term offering # term_offered = None # for _, unique_course in self.ds.unique_courses.iterrows(): # if row['Course #'] == unique_course['Course #']: # term_offered = unique_course['Term Offered'] # break # Run histogram generation and saving self.gen_and_save_histograms(dirs_and_files=found_grade_files, dir_priorities=search_list, indicator_data=indicator_data, bins=bins, row=row, rownum=rownumber, program=program, missing_data=missing_data) # # If the lists are empty, add data entry to missing data file # if not any(found_grade_files[k] for k in found_grade_files.keys()): # missing_thing = "{a} {b} ({c}-{d})".format( # a=row["Course #"], # b=row["Method of Assessment"], # c=row['Indicator #'], # d=row['Level'] # ) # logging.warning("No data found for %s", missing_thing) # missing_data.write("Missing data for {}\n".format(missing_thing)) # #----------------------------------------------------------------------------- # # Set up the found_grade_files dictionary in a way that the program will # # recognize as missing data and generate a blank histogram for # #----------------------------------------------------------------------------- # found_grade_files = {program: [None]} # else: # logging.debug("Found files in %s", ', '.join(found_grade_files.keys())) # #----------------------------------------------------------------------------- # # Iterate across the list of grade files found and separately generate a # # histogram for each file. Referring to the grade file keys as the location # # at which the grades file was found. Generates a blank histogram in cases # # where no data was found. # #----------------------------------------------------------------------------- # for location in found_grade_files.keys(): # for file in range(0, len(found_grade_files[location])): # # Provide DataFrame to generate empty histograms if no data is found # if isinstance(found_grade_files[location][file], type(None)): # grades = pd.DataFrame({'NO DATA FOUND': [-1]}) # else: # Open the grades normally # grades = pd.read_excel("{}/{}/{}".format( # self.config.grades_loc, # location, # found_grade_files[location][file] # ) # ) # # Ready the DataFrame for histogramming # grades = self.ready_DataFrame( # grades, # course=row['Course #'], # term_offered=term_offered # ) # #----------------------------------------------------------------------------- # # If the program that is being iterated across does not match the folder that # # the data was found, that should be indicated. The program entry in the # # header will always be based on which indicator lookup table the information # # came from. A note gets added to the header information in the case that data # # does not come from the same thing. Additionally, the program that the # # indicator belongs to will always get a copy of the file saved to their # # histgograms directory. # #----------------------------------------------------------------------------- # save_copies_to = [program] # if location != program: # indicator_data['Note'] = "Using data from {}".format(location) # logging.debug("Additional copy of this histogram being saved to %s", location) # save_copies_to.append(location) # # Generate the Report object # logging.debug("Generating histogram from %s/%s/%s", # self.config.grades_loc, # location, # found_grade_files[location][file] # ) # report = Report.generate_report(indicator_data, bins, self.config, grades) # #------------------------------------------------------------------------ # # Save the report with a file name reflective of the program, indicator # # and assessment type to the location that the data came from as well as # # the program subfolder that the indicator data belongs to. In the case # # Where multiple grade files were found, add a mumber to the file name. # #------------------------------------------------------------------------ # # File name formatted string setup # if len(save_copies_to) > 1: # # histogram_name = "{pth}/{saveloc}/{prgm} {ind}-{lvl} {crs} {asmt}_{i} {cfg}.pdf" # # File name that reflects row number in indicator sheet and just the indicator # histogram_name = "{pth}/{saveloc}/{prgm} {row} {ind}-{lvl} datafrom_{dataloc} {cfg}" # else: # # histogram_name= "{pth}/{saveloc}/{prgm} {ind}-{lvl} {crs} {asmt} {cfg}.pdf" # # File name that reflects row number in indicator sheet and just the indicator # histogram_name = "{pth}/{saveloc}/{prgm} {row} {ind}-{lvl} {cfg}" # # Save iteration # for i in range (0, len(save_copies_to)): # # File name formatted string formatting # save_as = histogram_name.format( # pth = self.config.histograms_loc, # saveloc = save_copies_to[i], # prgm = program, # ind = row["Indicator #"], # lvl = row["Level"][0], # crs = row["Course #"], # asmt = row["Method of Assessment"], # i = i, # cfg = self.config.name, # # row is rownumber + 2 because Pandas indexes from 0 starting from row 2 in # # the original spreadsheet. zfill adds padding zeros # row = str(rownumber + 2).zfill(3), # dataloc = location # ) # # Make sure the save directory exists, then save the file # os.makedirs("{pth}/{key}".format( # pth = self.config.histograms_loc, # key = save_copies_to[i] # ), # exist_ok=True # ) # report.save(save_as) logging.info("Autogeneration done!") def ready_DataFrame(self, grades, course, term_offered): """Pretty up a DataFrame for histogramming This method: * Turns DataFrame columns into cohort messages to act as legend entries * Filters cohorts if required * Indicates if a cohort has no data available The columns should look something like below normally: +--------+--------+--------+ | 201603 | 201703 | 201803 | +========+========+========+ | ... | ... | ... | +--------+--------+--------+ The method will turn these into cohort messages and use cohort filterings. If a requested cohort does not appear in the grades files, a No Data column will get added for that cohort. Args: grades(DataFrame): The grades that were just loaded course(string): The course number (e.g. 'ENGI 3821') term_offered(int): The term that the course is offered Returns: DataFrame: The prettied-up DataFrame """ # Try changing the the DataFrame columns to cohort messages. try: grades.columns = grades_org.cols_to_cohorts( grades=grades, course_name=course, course_term_offered=term_offered) # If the columns names were not integer-convertible, append size and move on except Exception: logging.debug("Grade columns for this assessment of %s were not integers. " \ + "Appending column sizes to original column names", course ) # Uses a list comprehension to append the column's true size to the message grades.columns = [ "{crs}({sz})".format(crs=entry, sz=grades_org.true_size(grades[entry])) for entry in grades.columns ] # If a cohort filter is in place, take the columns that have the correct cohort(s) if self.cohorts is not None: logging.debug("Detected a cohort filter for cohorts %s", ', '.join([str(c) for c in self.cohorts])) # Start a list to append relevant DataFrame columns to new_grades_cols = [] # Iterate across each cohort requested for c in self.cohorts: # Iterate across the grades DataFrame columns occurrences = 0 for col in grades.columns: if str(c) in col: new_grades_cols.append(grades[col]) occurrences += 1 logging.debug("Matched cohort %s with column %s", str(c), col) # If no matching columns are found, append an NDA column if occurrences is 0: logging.warning( "No grades columns were matched to cohort %s", str(c)) new_grades_cols.append( pd.Series([-1], name="Co{}: NDA".format(str(c)))) # Create a new DataFrame by concetenating all columns grades = pd.concat(new_grades_cols, axis=1, keys=[s.name for s in new_grades_cols]) return grades def gen_and_save_histograms(self, dirs_and_files, dir_priorities, indicator_data, bins, row, rownum, program, missing_data): """Generates and saves histograms for all files passed in Each file passed to gen_and_save_histograms will have a histogram generated for it and saved. Args: dirs_and_files(dict(list(string))): The grade files that were found. See the Examples section for some sample inputs dir_priorities(list(string)): The directory priority to which files should be saved. In cases where data is not found at the highest priority directory, but data is found at a lower priority directory, the highest priority directory will get a copy of the histogram(s) using the data sets in that low priority directory. indicator_data(dict(string)): The indicator data for generating Report objects. The method will pull other required items from here such as Program, Course Number, etc. bins(list(int or float)): The bins for generating Report objects row(pd.Series): The row that the program got all of this indicator data from rownum(int): The row number that all of this indicator data is coming from program(string): The program that all of this indicator data belongs to missing_data: The missing data file Examples: dirs_and_files comes in like this (and dir_priorities is in the same order as the keys):: { 'ENCM': ['MATH 1001 Course Grade - ENCM.xlsx'], 'Core': [ 'MATH 1001 Course Grade - Core.xlsx', 'MATH 1001 Course Grade - Core - By Semester.xlsx' ], 'ECE': ['MATH 1001 Course Grade - ECE.xlsx'] } Histograms get saved like so: * ENCM gets a histogram of the one ENCM file * Core gets 2 histograms of the Core files * ECE gets 1 histogram of the ECE file dirs_and_files comes in like this (and dir_priorities is in the same order as the keys):: { 'ENCM': [], 'Core': ['MATH 1001 Course Grade - Core.xlsx'], 'ECE': [] } Histograms get saved like so: * ENCM gets a copy of a histogram using the data from Core * Core gets a copy of a histogram using their data, but ENCM's indicator information * ECE gets nothing saved to it dirs_and files comes in like this (and dir_priorities is in the same order as the keys):: { 'ENCM': [], 'Core': [ 'MATH 1001 Course Grade - Core.xlsx', 'MATH 1001 Course Grade - Core - Program Comparison.xlsx' ], 'ECE': ['MATH 1001 Course Grade - ECE.xlsx'] } Histograms get saved like so: * ENCM gets two histograms, both copies of the ones using data from Core * Core gets two histograms using their data, but with ENCM's indicator information * ECE gets one histogram using their data, but with ENCM's indicator information dirs_and_files comes in like this (and dir_priorities is in the same order as the keys):: { 'ENCM': [], 'Core': [], 'ECE': [] } Histograms get saved like so: * Only ENCM gets a histogram. That histogram indicates that no data was found """ logging.info("Preparing to save and generate a list of histograms") logging.debug("dirs_and_files is %s", str(dirs_and_files)) logging.debug("dir_priorities is %s", str(dir_priorities)) # Keep track of the number of locations that come up empty empty_locations = 0 # Indicates if an extra copy of the histogram needs to be saved to the top priority dir save_extra_to = "" if dirs_and_files[ dir_priorities[0]] == []: # If the top priority dir is empty logging.debug( "First priority dir %s is empty. Will save a histogram there", dir_priorities[0]) save_extra_to = dir_priorities[0] # Check the unique course table to see if the course has a unique term offering term_offered = None for _, unique_course in self.ds.unique_courses.iterrows(): if row['Course #'] == unique_course['Course #']: term_offered = unique_course['Term Offered'] logging.info("Course %s came up with unique term_offered %s", row['Course #'], str(term_offered)) break for dir in dir_priorities: logging.debug("Generating histograms from dir %s", dir) if dirs_and_files[dir] == []: logging.debug("Dir %s is empty. Moving on...", dir) empty_locations += 1 logging.debug("empty_locations is %i", empty_locations) else: # If the dir does not match the program that the indicator belongs to, indicate that if dir != program: indicator_data['Notes'] = "Using data from {}".format(dir) for file in dirs_and_files[ dir]: # For each grades file in the specified dir # Open the grades file logging.debug( "Reading file %s", "{}/{}/{}".format(self.config.grades_loc, dir, file)) grades = pd.read_excel("{}/{}/{}".format( self.config.grades_loc, dir, file)) # Ready the DataFrame for histogramming grades = self.ready_DataFrame(grades, course=row['Course #'], term_offered=term_offered) # Generate a Report object rprt = Report.generate_report(indicator_data, bins, self.config, grades) # File name format if the list of files is only one long if len(dirs_and_files[dir]) == 1: histogram_name = "{program} {row} {ind}-{lvl} {cfg}" # File name format if the list of files is longer else: histogram_name = "{program} {row} {ind}-{lvl}_{i} {cfg}" # Save the Report to the current directory save_name = histogram_name.format( program=program, ind=row["Indicator #"], lvl=row["Level"][0], i=dirs_and_files[dir].index(file), cfg=self.config.name, # row is rownumber + 2 because Pandas indexes from 0 starting from row 2 in # the original spreadsheet. zfill adds padding zeros row=str(rownum + 2).zfill(3)) # Make sure the save directory exists, then save the file os.makedirs("{path}/{dir}".format( path=self.config.histograms_loc, dir=dir), exist_ok=True) rprt.save("{path}/{dir}/{name}".format( path=self.config.histograms_loc, dir=dir, name=save_name)) # If the extra save location was specified, save a copy there if save_extra_to: os.makedirs("{path}/{dir}".format( path=self.config.histograms_loc, dir=program), exist_ok=True) rprt.save("{path}/{dir}/{name}".format( path=self.config.histograms_loc, dir=save_extra_to, name=save_name)) # Set the extra save location back to an empty string to prevent further saving # Happens outside of the "files in directory" for-loop so that the extra location # gets copies of every file that matches the criteria. To prevent that behaviour, # indent by one. save_extra_to = "" # If all locations came up empty, generate a blank histogram in the Core directory if empty_locations == len(dir_priorities): logging.debug("empty_locations = %i while len(dir_priorities = %i", empty_locations, len(dir_priorities)) # Write the missing data entry to the missing data file missing_thing = "{a} {b} ({c}-{d})".format( a=row["Course #"], b=row["Method of Assessment"], c=row['Indicator #'], d=row['Level']) logging.warning("No data found for %s", missing_thing) missing_data.write("Missing data for {}\n".format(missing_thing)) # Set up blank histogram Report grades = pd.DataFrame({'NO DATA FOUND': [-1]}) indicator_data[ 'Notes'] = 'No file containing any of this assessment data was found' rprt = Report.generate_report(indicator_data, bins, self.config, grades) # Set up file name histogram_name = "{program} {row} {ind}-{lvl} {cfg} NDA" save_name = histogram_name.format( program=program, ind=row["Indicator #"], lvl=row["Level"][0], cfg=self.config.name, # row is rownumber + 2 because Pandas indexes from 0 starting from row 2 in # the original spreadsheet. zfill adds padding zeros row=str(rownum + 2).zfill(3)) # Check to see that the directory exists os.makedirs("{path}/{dir}".format(path=self.config.histograms_loc, dir=program), exist_ok=True) rprt.save("{path}/{dir}/{name}".format( path=self.config.histograms_loc, dir=program, name=save_name)) # Save histogram to program directory rprt.save("{path}/{dir}/{name}".format( path=self.config.histograms_loc, dir=program, name=save_name)) logging.info( "Finished generating histograms for row %s of %s indicators", str(rownum), str(program)) @staticmethod def autogenerate( config, programs=None, cohorts=None, whitelist=None, ds=None, indicators_loc=None, grades_loc=None, histograms_loc=None, ): """Shortcut to start histogram generation Initializes a ReportGenerator object and immediately starts report generation. TODO: * Create the method """ pass
class Downloader(object): #建立类成员 def __init__(self): #实例化同级类 self.parser = DataParser() self.datastore = DataStore() # 1.通用的网页请求抓取 def get_html(self, url): try: # 使用随机User-Agent ua = UserAgent() req_headers = {'User-Agent': ua.random} res = requests.get(url, headers=req_headers) if res.status_code == requests.codes.ok: html = res.text return html return '' except Exception as e: return e # 2.下载指定作者的所有作品 def downloadworks_oneauthor(self, start_url, authorinfotuple): # 1)提取作者信息,并设置请求的完整url和结果记录文件名 pagenum = 1 authorid = authorinfotuple[0] authorname = authorinfotuple[1] # 2)组成目标页面URL,循环爬行当前作者全部诗文 personalworks_hommeurl = start_url + 'page=%s&id=%s' % (str(pagenum), authorid) # 3)遍历所有页面,下载并保存到文件中 try: # i.爬取个人作品首页,提取总页数 works_html = self.get_html(personalworks_hommeurl) pagecount = self.parser.getpagecount(works_html) # ii.创建文档,写入基本信息 totalinfo = u'\n作者:{name},页数:{pagecount}\r\n'.format( name=authorname, pagecount=pagecount) path = u'作品集' filename = path + '\\' + authorname + '.txt' self.datastore.createfile_oneauther(filename, path, totalinfo) # iii.遍历作者所有作品页,提取诗文保存到指定文档 for i in range(1, pagecount + 1): #组合每一页的url page_url = start_url + 'page=%s&id=%s' % (str(i), authorid) #请求抓取当前诗文页面 time.sleep(random.randint(3, 6)) singlepageworks_html = self.get_html(page_url) if len(works_html) > 0: # 提取当前页中所有诗文 titlelist, contentlist = self.parser.getworks_singlepage( singlepageworks_html) # 写入文档 self.datastore.storeworks_singlepage( filename, i, titlelist, contentlist) return 'finished' except Exception as e: return e
from datetime import datetime import cPickle as pk import json from DataStore import DataStore # from FHMM import FHMM # from HMM import HMM, HMM_MAD from Preprocessing import Appliance, train_test_split, create_matrix with open('aws_keys.json') as f: data = json.load(f) ACCESS_KEY = data['access-key'] SECRET_ACCESS_KEY = data['secret-access-key'] DStore = DataStore('ngalvbucket1', 'house_1') all_channels = [1, 12, 5, 3, 10, 6, 9, 43, 7, 8] select_channels = [12, 5, 3, 10, 6, 9, 43, 8] # select_channels = [12, 5, 6] DStore.create_store(all_channels) top_10 = DStore.select_top_k(10,'2013-08-01','2013-09-01') combined = DStore.create_combined_df('2013-06-01 00:00:00', '2013-10-31 23:59:59', select_channels = select_channels, freq='1Min') with open('combined.pkl', 'w') as f: pk.dump(combined,f)