class MainWindow:
def __init__(self):
self.main_win = QMainWindow()
self.ui = Ui_MainWindow()
self.ds = DataStore()
self.ui.setupUi(self.main_win)
self.ui.stackedWidget.setCurrentWidget(self.ui.home)
self.ui.bmr_batton.clicked.connect(self.showBMR)
self.ui.bmi_button.clicked.connect(self.showBMI)
self.ui.bmr_calculate_button.clicked.connect(self.calcBMR)
self.ui.bmi_calculate_button.clicked.connect(self.calcBMI)
def show(self):
self.main_win.show()
def showBMR(self):
self.ui.stackedWidget.setCurrentWidget(self.ui.bmr)
def showBMI(self):
self.ui.stackedWidget.setCurrentWidget(self.ui.bmi)
def calcBMR(self):
if self.ui.bmr_male_radio.isChecked():
gender = "male"
else:
gender = "female"
height = int(self.ui.bmr_height_input.text())
weight = int(self.ui.bmr_weight_input.text())
age = int(self.ui.bmr_age_input.text())
if self.ui.bmr_sedentary.isChecked():
activity = "sedentary"
elif self.ui.bmr_lighthy.isChecked():
activity = "lighthy"
elif self.ui.bmr_moderately.isChecked():
activity = "moderately"
elif self.ui.bmr_very.isChecked():
activity = "very"
else:
activity = "extremely"
result = self.ds.bmr_calc(gender, height, weight, age, activity)
output = "The BMR for a {} year old {}, who is {} cm tall and {} kg heavy with a {} activity level is {} calories.".format(
age, gender, height, weight, activity, result)
self.ui.bmr_output_label.setText(output)
def calcBMI(self):
height = int(self.ui.bmi_height_input.text())
weight = int(self.ui.bmi_weight_input.text())
result = self.ds.bmi_calc(height, weight)
outpuut = "You entered the following information for BMI: Height {} cm & Weight: {} kg. This meansyourBMI is {}".format(
height, weight, result)
self.ui.bmi_output_label.setText(outpuut)
def __init__(self, str_id, df=None, rematch=False):
self.df = df
self.str_id = str_id
self.initialize()
self.ds = DataStore()
if rematch:
self.ds.rematch = True
def __init__(self):
self.main_win = QMainWindow()
self.ui = Ui_MainWindow()
self.ds = DataStore()
self.ui.setupUi(self.main_win)
self.ui.stackedWidget.setCurrentWidget(self.ui.home)
self.ui.bmr_batton.clicked.connect(self.showBMR)
self.ui.bmi_button.clicked.connect(self.showBMI)
self.ui.bmr_calculate_button.clicked.connect(self.calcBMR)
self.ui.bmi_calculate_button.clicked.connect(self.calcBMI)
def __init__(self,
rand_seed,
display=False,
no_op_max=27,
testing=False,
show_trades=None):
#self.ale = ALEInterface()
np.random.seed(rand_seed)
self._no_op_max = no_op_max
self.sequence_length = SEQUENCE_LENGTH
features_list = FEATURES_LIST
self.features_length = len(FEATURES_LIST)
path = STORE_PATH
training_days = TRAINING_DAYS
testing_days = TESTING_DAYS
# Load the data
training_store = DataStore(training_days=training_days,
features_list=features_list,
sequence_length=self.sequence_length)
if testing:
print("Set up for testing")
testing_store = DataStore(training_days=training_days,
testing_days=testing_days,
features_list=features_list,
sequence_length=self.sequence_length,
mean=training_store.mean,
std=training_store.std)
self.environment = Trading(data_store=testing_store,
sequence_length=self.sequence_length,
features_length=self.features_length,
testing=testing,
show_trades=show_trades)
else:
self.environment = Trading(data_store=training_store,
sequence_length=self.sequence_length,
features_length=self.features_length,
testing=testing,
show_trades=show_trades)
self.old_x = 0.
# collect minimal action set
#self.real_actions = self.ale.getMinimalActionSet()
self.real_actions = [0, 1, 2]
# height=210, width=160
self.reset()
def __init__(self,
rand_seed,
display=False,
no_op_max=27,
testing=False,
show_trades=None):
np.random.seed(rand_seed)
self._no_op_max = no_op_max
self.sequence_length = SEQUENCE_LENGTH
features_list = FEATURES_LIST
self.features_length = len(FEATURES_LIST)
path = STORE_PATH
training_days = TRAINING_DAYS
testing_days = TESTING_DAYS
# Load the data
training_store = DataStore(training_days=training_days,
sequence_length=self.sequence_length)
if testing:
print("Set up for testing")
testing_store = DataStore(training_days=training_days,
testing_days=testing_days,
sequence_length=self.sequence_length,
mean=training_store.mean,
std=training_store.std)
self.environment = Trading(data_store=testing_store,
sequence_length=self.sequence_length,
features_length=self.features_length,
testing=testing,
show_trades=show_trades)
else:
self.environment = Trading(data_store=training_store,
sequence_length=self.sequence_length,
features_length=self.features_length,
testing=testing,
show_trades=show_trades)
self.old_x = 0.
self.reset()
def randomize(graph):
i = 0
for n in graph.nodes():
i += 1
randomloc = float(random.randint(0, 999999999)) / 1000000000
newnode = (randomloc, )
neighbors = graph.neighbors(n)
graph.add_node(newnode, id=i, ds=DataStore(100000))
for neighbor in neighbors:
graph.add_edge(newnode, neighbor)
graph.remove_node(n)
def choose_file_rating(folder="data/match_rating/"):
ds = DataStore()
files = os.listdir(folder)
files.sort(key=lambda f: int(re.sub('\D', '', f)))
print("Welcome to the analyzer. Which file would you like to analyze?")
for index, file in enumerate(files):
print("({}) {}".format(index, file))
file_index = input(
"Write the number to choose the file. Write anything else to cancel\n")
if int(file_index) in range(1, len(files)):
print("File {} chosen".format(files[int(file_index)]))
results = ds.load_rating_file(files[int(file_index)])
return results
else:
print("Command not recognized. Returning.")
return
def calculate_all_ratings(folder="data/match_rating/"):
ds = DataStore()
files = os.listdir(folder)
files.sort(key=lambda f: int(re.sub('\D', '', f)))
for file in files:
print("File {} chosen".format(file))
results = ds.load_rating_file(file)
# calculate the relevant data
count_maybe = 0
count_yes = 0
count_no = 0
for i, result in results.iterrows():
rating = result["match_rating"]
if rating == 0:
count_no += 1
elif rating == 1:
count_yes += 1
elif rating == -1:
count_maybe += 1
# Create data
data = {"bad": count_no, "good": count_yes, "maybe": count_maybe}
output_path = "data/rating_result.csv"
now_id = file.replace("rating_", "").replace(".csv", "")
data["id"] = now_id
if os.path.exists(output_path):
df = pd.read_csv(output_path)
df2 = pd.DataFrame([data])
df3 = df.append(df2)
df3.to_csv(output_path, index=False, mode='w')
else:
df = pd.DataFrame([data])
df.to_csv(output_path, index=False, mode='w')
print("Good: {}\nBad: {}\nMaybe: {}".format(count_yes, count_no,
count_maybe))
def __init__(self):
#实例化其他几个功能模块
self.urlgen = URLGenerator()
self.downloader = Downloader()
self.parser = DataParser()
self.datastore = DataStore()
#建立3个url队列：未下载、正在下载、完成下载
self.beforedownloadset = set()
self.beingdownloadset = set()
self.afterdownloadset = set()
#设定种子url
self.seedurl = 'https://so.gushiwen.org/authors/'
def get_risk_map(self, location=None, box=None, name="map", accuracy=10):
if location is not None:
locs = location.split(',')
resp = requests.get(self._BASE_URL.format(
locs[0], locs[1])).json()["boundingbox"]
box = resp[0] + "," + resp[2] + "," + resp[1] + "," + resp[3]
coords = str(box).split(',')
first = Location(float(coords[0]), float(coords[1]))
second = Location(float(coords[2]), float(coords[3]))
locParser = LocationParser(first, second, accuracy=accuracy)
locParser.parse()
elevParser = ElevationParser(locParser.grid)
DataStore.save(elevParser.grid, name=name)
data = DataStore.load(name)
s = getWatershed(data, 100)
m = RiskMap(s, 10)
m.generate_kml(name=name)
def main():
if (len(sys.argv) > 1):
if sys.argv[1] == "check":
idl_path = parse(sys.argv[2:], 'i:', ["idl="])[0]
#To do: call api
dds = DDSConnector(idl_path)
if dds.IsIDLValid():
print("valid")
else:
print("invalid")
elif sys.argv[1] == "run":
config = configparser.ConfigParser()
config.read_file(open(parse(sys.argv[2:], 'c:', ["config="])[0]))
if config['common']['converter'] == 'dds':
if config['dds']['mode'] == 'subscriber':
dtStore = DataStore(db_name=config['common']['package_id'])
dtStore.AddDataToCollection(
"file_idl",
json_obj={
"id":
dtStore.AddFileToDB(config['dds']['file_idl'])
})
dtStore.AddDataToCollection(
"network_config",
json_obj={
"id":
dtStore.AddFileToDB(
config['dds']['network_config'])
})
ckan = CKANConnector(config['common']['ckan_sever'],
config['common']['api_token'],
config['common']['resource_id'],
config['common']['package_id'])
ckan.CreateResource(config['dds']['file_idl'],
"The idl file")
ckan.CreateResource(config['dds']['network_config'],
"The network config file")
x = threading.Thread(target=ReceiveStreamData,
args=[dtStore, ckan],
daemon=True)
x.start()
dds = DDSConnector(config['dds']['file_idl'],
config['dds']['topic_name'],
config['dds']['network_config'],
config['dds']['mode'])
dds.Build()
dds.Run()
elif sys.argv[1] == "publish_example":
DDSConnector().PublishExample()
else:
print(sys.argv)
print(USAGE)
else:
print(sys.argv)
print(USAGE)
def choose_file(folder="data/matching_result/"):
ds = DataStore()
mentors, mentees, person_dict = ds.load_data("initial_data_{}".format(
global_vars.ROUND))
files = os.listdir(folder)
files.sort(key=lambda f: int(re.sub('\D', '', f)))
print("Welcome to the analyzer. Which file would you like to analyze?")
for index, file in enumerate(files):
print("({}) {}".format(index, file))
file_index = input(
"Write the number to choose the file. Write anything else to cancel\n")
if int(file_index) in range(1, len(files)):
print("File {} chosen".format(files[int(file_index)]))
results = ds.load_result_file(files[int(file_index)])
str_id = files[int(file_index)].replace("matching_result_",
"").replace(".csv", "")
else:
print("Command not recognized. Returning.")
return
return person_dict, results, str_id
def __init__(self):
data_sources = DataSources(authors="data/author.csv",
books="data/book.csv",
books_to_subjects="data/book2subjects.csv",
subjects="data/subject.csv",
sub_subjects="data/sub_subject.csv",
sub_sub_subjects="data/sub_sub_subject.csv")
self._data_store = DataStore(data_sources)
self._excel = ExcelReportGenerator(self._data_store)
self._controller = Controller(self._data_store, self._excel)
self._window = AppWindow(self._controller)
sys.exit(self._app.exec_())
def hungarian_main(self):
global ds
ds = DataStore()
mentors, mentees = self.create_assignment_matrix()
self.matrix_size = len(self.score_matrix)
aux_row_vars = self.init_row_aux_vars()
aux_col_vars = self.init_col_aux_vars()
# Master method for the hungarian algorithm / job assignment
self.hungarian_algorithm(aux_row_vars, aux_col_vars, 1)
matching_list = self.format_matching_result(mentors, mentees)
# ds.save_matches(matching_list)
show_matches(matching_list)
return matching_list
for f in next(os.walk(v_path), (None, None, []))[2]:
os.remove(os.path.join(v_path, f))
# Training sets
"""input_text = "abcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddef"\
+ "abcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddefabcddef"""""
#input_text = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, ipsum sed diam"
#with os.open("lstm_training_data/vanishing_vocables_de.txt", "r"):
#with os.open("lstm_training_data/loremipsum.txt", "r"):
with os.fdopen(os.open("lstm_training_data/schachnovelle.txt", os.O_RDONLY)) as f:
input_text = f.read()
# initialize data store object
data_store = DataStore(
input_data_set=set(input_text),
output_data_set=set(input_text),
input_data=input_text)
# set data_store configuration
data_store.configure({
"sequence_length": config["time_steps"]
})
# samples is a generator object for sequences of length data_store.config['time_steps']
samples = data_store.samples()
# Uncomment to use random samples for training. The amount specifies the number of sequences
# to be trained during one iteration. After each iteration the sequences will be shuffled again.
#samples = data_store.random_samples(amount=20)
lstm = LSTMNetwork()
def __init__(self):
value = {
"Command": "INIT",
"Token": "8051bf89-e115-4147-8e5a-ff9d6f39f0d7"
}
headers = {
'Content-type': 'application/json',
'Accept': 'application/json',
}
jvalue = json.dumps(value)
conn = httplib.HTTPConnection('107.20.243.77', 80)
conn.request('POST', '/api/hermes', jvalue, headers)
response = conn.getresponse()
ret = json.loads(str((response.status, response.reason, response.read())[2]))
conn.close()
DS = DataStore()
ctrl = controller()
myFile = open('output.txt', 'w')
myFile.write(str(ret))
myFile.close()
looptime = ret['ServerState']['ServerTiers']['DB']['ServerStartTurnTime']
coef = (ret["ServerState"]["CostPerServer"] / ret["ServerState"]["ProfitConstant"])
DS.setCoef(coef)
infra = False
p = None
research = None
didGrid = False
progression = [None, "GRID", "GREEN", None ]
#while ret['ServerState']['TurnNo'] < 10080:
while True:
x = 0
while x <=looptime-1:
value = {
"Command": "PLAY",
"Token": "8051bf89-e115-4147-8e5a-ff9d6f39f0d7"
}
headers = {
'Content-type': 'application/json',
'Accept': 'application/json',
}
jvalue = json.dumps(value)
conn = httplib.HTTPConnection('107.20.243.77', 80)
conn.request('POST', '/api/hermes', jvalue, headers)
response = conn.getresponse()
ret = json.loads(str((response.status, response.reason, response.read())[2]))
#Get demand from server
demand = [ret['ServerState']['ServerTiers']['DB']['ServerRegions']['NA']['NoOfTransactionsInput']]
demand.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['EU']['NoOfTransactionsInput'])
demand.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['AP']['NoOfTransactionsInput'])
config = [ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['NA']['NodeCount']]
config.append(ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['EU']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['AP']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['JAVA']['ServerRegions']['NA']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['JAVA']['ServerRegions']['EU']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['JAVA']['ServerRegions']['AP']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['NA']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['EU']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['AP']['NodeCount'])
DS.avgDemand(demand)
DS.setConfig(config)
conn.close()
x+=1
lastProfit = ret["ServerState"]["ProfitEarned"]
GridCost = ret["ServerState"]["ResearchUpgradeLevels"][1]["UpgradeCost"]
GridTurns = ret["ServerState"]["ResearchUpgradeLevels"][1]["NoOfTurnsRequired"]
GridTotalCost = GridTurns * GridCost
if ret['ServerState']['TurnNo'] <= 9000 and ret["ServerState"]["ProfitAccumulated"] >= GridTotalCost/10 and GridCost < (lastProfit - (lastProfit/3)):
didGrid = True
try:
if ret["ServerState"]["ResearchUpgradeState"]["GRID"] == -1:
#research = "GREEN"
pass
except:
research = "GRID"
#p = research
#Calculate free space
#AVERAGE CAPACITY
# capacity = [ ( ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2]
# capacity.append(( ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2)
# capacity.append(( ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2)
#97%
capacity = [ ( ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']+ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] ) / 3]
capacity.append(( ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] ) / 3)
capacity.append(( ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit']) / 3)
#93%
# capacity = [ ( ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']+ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 3]
# capacity.append(( ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 3)
# capacity.append(( ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']) / 3)
#100% CAPACITY
# capacity = [ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit']]
# capacity.append(ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'])
# capacity.append(ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'])
# #90% CAPACITY
# capacity = [ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']]
# capacity.append(ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'])
# capacity.append(ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'])
DS.setCapacity(capacity)
changes = ctrl.calc(DS)
jsonchange = {"Servers":{
"WEB":{
"ServerRegions":{
"AP":{
"NodeCount":changes[2]
},
"EU":{
"NodeCount":changes[1]
},
"NA":{
"NodeCount":changes[0]
}
}
},
"JAVA":{
"ServerRegions":{
"NA":{
"NodeCount":changes[3]
},
"EU":{
"NodeCount":changes[4]
},
"AP":{
"NodeCount":changes[5]
}
}
},
"DB":{
"ServerRegions":{
"NA":{
"NodeCount":changes[6]
},
"EU":{
"NodeCount":changes[7]
},
"AP":{
"NodeCount":changes[8]
}
}
}
},
"UpgradeInfraStructure": infra,
"UpgradeToResearch": research
}
if research != None:
research = None
value = {
"Command": "CHNG",
"Token": "8051bf89-e115-4147-8e5a-ff9d6f39f0d7",
"ChangeRequest": jsonchange
}
headers = {
'Content-type': 'application/json',
'Accept': 'application/json',
}
jvalue = json.dumps(value)
conn = httplib.HTTPConnection('107.20.243.77', 80)
conn.request('POST', '/api/hermes', jvalue, headers)
response = conn.getresponse()
ret = json.loads(str((response.status, response.reason, response.read())[2]))
# if research != None:
# research = None
# jsonchange = {"Servers":{
# "WEB":{"UpgradeToResearch": "Green"}}}
# value = {
# "Command": "CHNG",
# "Token": "8051bf89-e115-4147-8e5a-ff9d6f39f0d7",
# #"Token": "7440b0b0-c5a2-4ab3-bdc3-8935865bb9d1",
# "ChangeRequest": jsonchange
# }
# headers = {'Content-type': 'application/json','Accept': 'application/json',}
# jvalue = json.dumps(value)
# conn = httplib.HTTPConnection('107.20.243.77', 80)
# #conn = httplib.HTTPConnection('uat.hermes.wha.la', 80)
# conn.request('POST', '/api/hermes', jvalue, headers)
# response = conn.getresponse()
# ret = json.loads(str((response.status, response.reason, response.read())[2]))
#play
value = {
"Command": "PLAY",
"Token": "8051bf89-e115-4147-8e5a-ff9d6f39f0d7"
}
headers = {
'Content-type': 'application/json',
'Accept': 'application/json',
}
jvalue = json.dumps(value)
conn = httplib.HTTPConnection('107.20.243.77', 80)
conn.request('POST', '/api/hermes', jvalue, headers)
response = conn.getresponse()
ret = json.loads(str((response.status, response.reason, response.read())[2]))
#Get demand from server
demand = [ret['ServerState']['ServerTiers']['DB']['ServerRegions']['NA']['NoOfTransactionsInput']]
demand.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['EU']['NoOfTransactionsInput'])
demand.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['AP']['NoOfTransactionsInput'])
config = [ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['NA']['NodeCount']]
config.append(ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['EU']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['AP']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['JAVA']['ServerRegions']['NA']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['JAVA']['ServerRegions']['EU']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['JAVA']['ServerRegions']['AP']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['NA']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['EU']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['AP']['NodeCount'])
DS.resetDemand(demand)
DS.setConfig(config)
coef = (ret["ServerState"]["CostPerServer"] / ret["ServerState"]["ProfitConstant"])
DS.setCoef(coef)
# f = open('myfile.txt','w')
# f.write(str(ret)) # python will convert \n to os.linesep
# f.close()
conn.close()
print 'Turn: ' + str(ret['ServerState']['TurnNo'])
print "WEB capacity: " + str(capacity[0])
print "JAVA capacity: " + str(capacity[1])
print "DB capacity: " + str(capacity[2])
print "ServerCost: " + str(ret["ServerState"]["CostPerServer"])
#print didGrid
#if didGrid:
try:
inf = str(ret["ServerState"]["InfraStructureUpgradeState"]["Value"])
if inf >=0:
print "INFRA value: " + inf
except:
pass
try:
grid = str(ret["ServerState"]["ResearchUpgradeState"]["GRID"])
if grid != "-1":
print "---Researching: "+ "GRID" +"---\nTurns Left: " + grid
if int(grid) <= 1441 and int(grid) >= 1430:
#infra = True
pass
else:
infra = False
else:
print "GRID UPGRADE COMPLETE"
except:
pass
try:
green = str(ret["ServerState"]["ResearchUpgradeState"]["GREEN"])
if green != "-1":
print "---Researching: "+ "GREEN" +"---\nTurns Left: " + green
else:
print "GREEN UPGRADE COMPLETE"
except:
pass
print demand
print ' ' + str(config[0]) + ' ' + str(config[1]) + ' ' + str(config[2]) + ' ' + '\n ' + str(config[3]) + ' ' + str(config[4]) + ' ' + str(config[5]) + ' ' + '\n ' + str(config[6]) + ' ' + str(config[7]) + ' ' + str(config[8])
print ''
conn.close()
# resp = requests.get(_BASE_URL.format(21.82706108, 75.613033)).json()["boundingbox"]
# box = resp[0] + "," + resp[2] + "," + resp[1] + "," + resp[3]
# coords = str(box).split(',')
# first = Location(float(coords[0]), float(coords[1]))
# second = Location(float(coords[2]), float(coords[3]))
first_lat = float(input("Enter first latitude:"))
first_lon = float(input("Enter first longitude:"))
second_lat = float(input("Enter second latitude:"))
second_lon = float(input("Enter second longitude:"))
first = Location(first_lat, first_lon)
second = Location(second_lat, second_lon)
name = input("Enter city name:")
accuracy = int(input("Enter accuracy in meteres:"))
locParser = LocationParser(first, second, accuracy=accuracy)
locParser.parse()
elevParser = ElevationParser(locParser.grid)
DataStore.save(elevParser.grid, name=name)
data = DataStore.load(name)
s = getWatershed(data, 100)
m = RiskMap(s, 10)
m.generate_kml(name=name)
def __init__(self):
tok = "8051bf89-e115-4147-8e5a-ff9d6f39f0d7"
url = "hermes.wha.la"
value = {
"Command": "INIT",
"Token": tok
}
headers = {
'Content-type': 'application/json',
'Accept': 'application/json',
}
jvalue = json.dumps(value)
conn = httplib.HTTPConnection(url, 80)
conn.request('POST', '/api/hermes', jvalue, headers)
response = conn.getresponse()
ret = json.loads(str((response.status, response.reason, response.read())[2]))
conn.close()
DS = DataStore(ret["ServerState"]["ServerTiers"]["DB"]["ServerStartTurnTime"])
ctrl = controller()
# myFile = open('output.txt', 'w')
# myFile.write(str(ret))
# myFile.close()
looptime = ret['ServerState']['ServerTiers']['DB']['ServerStartTurnTime']
coef = (ret["ServerState"]["CostPerServer"] / ret["ServerState"]["ProfitConstant"])
WebRefresh = ret["ServerState"]["ServerTiers"]["WEB"]["ServerStartTurnTime"]
JavaRefresh = ret["ServerState"]["ServerTiers"]["JAVA"]["ServerStartTurnTime"]
DBRefresh = ret["ServerState"]["ServerTiers"]["DB"]["ServerStartTurnTime"]
DS.setCoef(coef)
infra = False
p = None
research = None
didGrid = False
progression = [None, "GRID", "GREEN", None ]
#while ret['ServerState']['TurnNo'] < 10080:
while True:
x = 0
value = {
"Command": "PLAY",
"Token": tok
}
headers = {
'Content-type': 'application/json',
'Accept': 'application/json',
}
jvalue = json.dumps(value)
conn = httplib.HTTPConnection(url, 80)
conn.request('POST', '/api/hermes', jvalue, headers)
response = conn.getresponse()
ret = json.loads(str((response.status, response.reason, response.read())[2]))
turnnumber = ret['ServerState']['TurnNo']
#Get demand from server
demand = [ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['NA']['NoOfTransactionsInput']]
demand.append(ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['EU']['NoOfTransactionsInput'])
demand.append(ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['AP']['NoOfTransactionsInput'])
config = [ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['NA']['NodeCount']]
config.append(ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['EU']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['AP']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['JAVA']['ServerRegions']['NA']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['JAVA']['ServerRegions']['EU']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['JAVA']['ServerRegions']['AP']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['NA']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['EU']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['DB']['ServerRegions']['AP']['NodeCount'])
# if turnnumber % (DBRefresh*2):
# DS.resetDemand(demand)
# else:
# DS.avgDemand(demand)
DS.runningDemand(demand)
DS.setConfig(config)
coef = (ret["ServerState"]["CostPerServer"] / ret["ServerState"]["ProfitConstant"])
DS.setCoef(coef)
conn.close()
lastProfit = ret["ServerState"]["ProfitEarned"]
GridCost = ret["ServerState"]["ResearchUpgradeLevels"][1]["UpgradeCost"]
GridTurns = ret["ServerState"]["ResearchUpgradeLevels"][1]["NoOfTurnsRequired"]
GridTotalCost = GridTurns * GridCost
#init research when its swaggin to do so
if ret['ServerState']['TurnNo'] <= 2000 and ret["ServerState"]["ProfitAccumulated"] >= GridTotalCost/8 and GridCost < (lastProfit - (lastProfit/3)):
didGrid = True
try:
if ret["ServerState"]["ResearchUpgradeState"]["GRID"] == -1:
#research = "GREEN"
pass
except:
research = "GRID"
pass
#p = research
#Calculate free space
#AVERAGE CAPACITY
# capacity = [ ( ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2]
# capacity.append(( ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2)
# capacity.append(( ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2)
#97%
capacity = [ ( ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']+ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] ) / 3]
capacity.append(( ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] ) / 3)
capacity.append(( ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit']) / 3)
#93%
# capacity = [ ( ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']+ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 3]
# capacity.append(( ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 3)
# capacity.append(( ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']) / 3)
#100% CAPACITY
# capacity = [ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit']]
# capacity.append(ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'])
# capacity.append(ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'])
# #90% CAPACITY
# capacity = [ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']]
# capacity.append(ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'])
# capacity.append(ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'])
DS.setCapacity(capacity)
webchanges = [0,0,0,0,0,0,0,0,0]
javachanges = [0,0,0,0,0,0,0,0,0]
dbchanges = [0,0,0,0,0,0,0,0,0]
if turnnumber%WebRefresh == 0:
webchanges = ctrl.calcWeb(DS,WebRefresh*1.5)
if turnnumber%JavaRefresh == 0:
javachanges = ctrl.calcJava(DS,JavaRefresh*1.5 )
if turnnumber%DBRefresh == 0:
dbchanges = ctrl.calcDB(DS,DBRefresh*1.5)
#create 'changes' to know what servers to bring up/down
changes = []
changes.append(webchanges[0])
changes.append(webchanges[1])
changes.append(webchanges[2])
changes.append(javachanges[3])
changes.append(javachanges[4])
changes.append(javachanges[5])
changes.append(dbchanges[6])
changes.append(dbchanges[7])
changes.append(dbchanges[8])
jsonchange = {"Servers":{
"WEB":{
"ServerRegions":{
"AP":{
"NodeCount":changes[2]
},
"EU":{
"NodeCount":changes[1]
},
"NA":{
"NodeCount":changes[0]
}
}
},
"JAVA":{
"ServerRegions":{
"NA":{
"NodeCount":changes[3]
},
"EU":{
"NodeCount":changes[4]
},
"AP":{
"NodeCount":changes[5]
}
}
},
"DB":{
"ServerRegions":{
"NA":{
"NodeCount":changes[6]
},
"EU":{
"NodeCount":changes[7]
},
"AP":{
"NodeCount":changes[8]
}
}
}
},
"UpgradeInfraStructure": infra,
"UpgradeToResearch": research
}
if research != None:
research = None
value = {
"Command": "CHNG",
"Token": tok,
"ChangeRequest": jsonchange
}
headers = {
'Content-type': 'application/json',
'Accept': 'application/json',
}
jvalue = json.dumps(value)
conn = httplib.HTTPConnection(url, 80)
conn.request('POST', '/api/hermes', jvalue, headers)
response = conn.getresponse()
ret2 = json.loads(str((response.status, response.reason, response.read())[2]))
conn.close()
#print Stuff
print 'Turn: ' + str(ret['ServerState']['TurnNo'])
print 'Total Profit: $' + str(ret["ServerState"]["ProfitAccumulated"])
# print "WEB capacity: " + str(capacity[0])
# print "JAVA capacity: " + str(capacity[1])
# print "DB capacity: " + str(capacity[2])
# print "ServerCost: " + str(ret["ServerState"]["CostPerServer"])
#print didGrid
#if didGrid:
try:
inf = str(ret["ServerState"]["InfraStructureUpgradeState"]["Value"])
if inf >=0:
#print "INFRA value: " + inf
pass
except:
pass
try:
grid = str(ret["ServerState"]["ResearchUpgradeState"]["GRID"])
if grid != "-1":
print "---Researching: "+ "GRID" +"---\nTurns Left: " + grid
if int(grid) <= 1441 and int(grid) >= 1430:
#infra = True
pass
else:
infra = False
else:
print "GRID UPGRADE COMPLETE"
except:
pass
try:
green = str(ret["ServerState"]["ResearchUpgradeState"]["GREEN"])
if green != "-1":
print "---Researching: "+ "GREEN" +"---\nTurns Left: " + green
else:
print "GREEN UPGRADE COMPLETE"
except:
pass
print demand
print ' ' + str(config[0]) + ' ' + str(config[1]) + ' ' + str(config[2]) + ' ' + '\n ' + str(config[3]) + ' ' + str(config[4]) + ' ' + str(config[5]) + ' ' + '\n ' + str(config[6]) + ' ' + str(config[7]) + ' ' + str(config[8])
print ''
conn.close()
id=1234,
gender="m",
linkedin="https://www.linkedin.com/in/christian-bøgelund-92747799/",
study_line="Cand. Alt",
university="DTU",
content=LOREM
)
mentees.append(mentee)
mentor = Mentor(
name="En helt anden mentor, der ikke er Nana",
age=0,
id=2345,
gender="m",
linkedin="https://www.linkedin.com/in/christian-bøgelund-92747799/",
study_line="Cand. Alt",
company="Microsoft",
university="",
company_type="",
position="CEO",
content=LOREM
)
mentors.append(mentor)
"""
# mentees, mentors = zip(*hungarian_main())
# show_matches(mentees, mentors)
ds = DataStore()
"""
# Temp import for testing
import sys
import os
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)) + '/..')
from Report import Report
from ReportConfig import ReportConfig
from DataStore import DataStore
import pandas as pd
import unsorted
import datetime
print("Loading ENCV indicators into DataStore")
ds = DataStore(['ENCV'])
print("Querying KB.1-D")
ds.query_indicators(program='ENCV', indicator='KB.1', level='D')
config = ReportConfig()
'''
print("Removing the NDA threshold")
config.NDA_threshold = 0.0
'''
for row in ds.last_query.iterrows():
print("Processing course", row[1]["Course #"])
indicator_data, bins = unsorted.parse_row(row[1], config.header_attribs)
indicator_data['Program'] = 'ENCV'
def __init__(
self,
config,
programs=None,
cohorts=None,
whitelist=None,
ds=None,
indicators_loc=None,
grades_loc=None,
histograms_loc=None,
):
"""Object Initialization
Args:
config(ReportConfig): A ReportConfig object
programs(string or list(string)): A list of programs to generate indicators for.
Defaults to using all programs. If only one string is passed in, it will
get put into a size-one list.
cohorts(int or list(int)): A list of cohorts to pull data from. Defaults to
using all cohorts. If only one cohort is passed in, it will get put into
a size-one list.
whitelist(dictionary): A dictionary of lists, keyed by the only stuff
to parse (e.g. 'course', 'indicator', etc.) and filled with the
specific values to uniquely parse. Defaults to no whitelist. The lists
in the dictionaries can be partial (i.e. if you pass 'KB' as part of the
'indicator' whitelist, it will pull all 'KB' indicators). If the
lists contain only one string, that string gets put into a
size-one list.
ds(DataStore): A DataStore object. Defaults to generating one based on the
whitelist entries for 'programs'
indicators_loc(string): The location of the indicator sheets. Defaults to
searching an "Indicators" folder in the directory above the project
(see ReportConfig for more info)
grades_loc(string): The location of the grades sheets. Defaults to searching
a "Grades" folder in the directory above the project (see
ReportConfig for more info)
histograms_loc(string): The location to store histograms. Defaults to using
a "Histograms" folder in the directory above the project (see
ReportConfig for more info)
"""
logging.info("Start of AutoGenerator initialization")
self.config = config
# logging.info("Initializing whitelist")
self.whitelist = whitelist
# Ensure that all whitelist entries are lists (if it exists, that is)
if self.whitelist:
logging.debug("Checking whitelist for list validity")
for entry in self.whitelist.keys():
self.whitelist[entry] = _check_list(self.whitelist[entry])
# if type(self.whitelist[entry]) is not type(list()):
# logging.debug("Whitelist entry %s is not a list. Converting to a one-size list", entry)
# # Change to one-size list
# self.whitelist[entry] = [self.whitelist[entry]]
# logging.info("Initializing cohorts list")
self.cohorts = cohorts
# Ensure that all cohorts entries are lists (if it exists, that is)
if self.cohorts:
logging.debug("Checking cohorts variable for list validity")
self.cohorts = _check_list(self.cohorts)
# logging.info("Initializing program list")
self.programs = programs
# Ensure list validity of programs
if self.programs:
logging.debug("Checking programs for list validity")
self.programs = _check_list(self.programs)
# Use all programs if none were provided
if not self.programs:
logging.debug(
"Programs not passed as paramater. Using list of all programs")
self.programs = globals.all_programs
# # Same check as whitelist - ensure that self.programs is a list
# if type(self.programs) is not type(list()):
# logging.debug("Programs was not passed in list format. Converting to a one-size list")
# self.programs = [self.programs]
# If any of the file location parameters were passed in, overwrite what ReportConfig has
if indicators_loc:
self.config.indicators_loc = indicators_loc
if grades_loc:
self.config.grades_loc = grades_loc
if histograms_loc:
self.config.histograms_loc = histograms_loc
logging.debug("Indicators location is %s", self.config.indicators_loc)
logging.debug("Grades location is %s", self.config.grades_loc)
logging.debug("Histograms location is %s", self.config.histograms_loc)
# Check to see if a DataStore was passed to init, create one if not
if not ds:
logging.debug(
"No DataStore object was passed to init; creating one now")
self.ds = DataStore(programs=self.programs,
indicators_loc=self.config.indicators_loc,
grades_loc=self.config.grades_loc)
# Make sure that the histograms folder exists
logging.info(
"Setting up output directories (Missing Data & Histograms)")
os.makedirs(os.path.dirname(__file__) + '/../Missing Data',
exist_ok=True)
os.makedirs(self.config.histograms_loc, exist_ok=True)
logging.info("ReportGenerator initialization done!")
def __init__(self):
#实例化同级类
self.parser = DataParser()
self.datastore = DataStore()
def __init__(self):
self.data_store = DataStore()
self.url_manager = UrlManager(self.data_store)
self.strategy_container = ParserStrategyContainer()
self.downloader = HtmlDownloader()
class BikeService:
"""A class used to represent the business logic and delegates actions to data layer."""
def __init__(self):
"""Default Constructor to initialize data/persistence layer and fetch records from csv."""
self.data_store = DataStore()
self.list_records = self.data_store.get_all_records()
def print_all(self):
"""Method to print all the records to console and every 9 records my name - Adam Mohr."""
count = 0
for row in self.list_records:
print(row)
count += 1
if count % 9 == 0:
self.print_adam()
self.print_length()
@staticmethod
def get_input_new_record():
"""Helper method to get input from user."""
year = input("Enter year: ")
sex = input("Enter sex: ")
age_group = input("Enter age group: ")
student_response = input("Enter student response: ")
vector = input("Enter vector: ")
coordinate = input("Enter coordinate: ")
value = input("Enter value: ")
new_record = BikeRecord(year, "Canada", "", sex, age_group,
student_response, "Percent", "239", "units",
"0", vector, coordinate, value, "", "", "",
"0")
return new_record
def insert_bike_record(self):
"""Method to insert new Bike Record at end of the list. Hardcoded certain values and left some fields blank
to match formatting found in original csv file."""
self.list_records.append(self.get_input_new_record())
print("Record added.")
@staticmethod
def print_adam():
"""Method to print the author's name - Adam Mohr."""
print("Program by: Adam Mohr")
def print_length(self):
"""Helper method to print length of list i.e. how many records."""
print(len(self.list_records), "Records found")
def save_records(self):
"""Method to save a new csv file to disk using DataStore method."""
self.data_store.save_records_to_disk()
def update_record(self):
"""Helper method to get valid list index number from user."""
self.print_length()
try:
index = int(
input("Enter index of record to update (starts at index 0): "))
self.list_records[index] = self.get_input_new_record()
print("Record index", index, "updated.")
except IndexError as e:
print(e, "please give valid index number (starts at index 0).")
except ValueError as e:
print(e, "please give valid integer number.")
def show_record(self):
"""Method to show a single record."""
self.print_length()
try:
index = int(
input("Enter index of record to show (starts at index 0): "))
print(self.list_records[index])
except IndexError as e:
print(e, "please give valid index number (starts at index 0).")
except ValueError as e:
print(e, "please give valid integer number.")
def delete_record(self):
"""Method to delete a single record."""
self.print_length()
try:
index = int(
input("Enter index of record to delete (starts at index 0): "))
self.list_records.pop(index)
print("Record index", index, "deleted.")
except IndexError as e:
print(e, "please give valid index number (starts at index 0).")
except ValueError as e:
print(e, "please give valid integer number.")
def sort_column(self):
"""Method to sort by column. Input is retrieved from user to determine which column to sort. Inline lambda
function is used with a key that identifies the sorting property."""
column = input(
"Enter column number to sort by: 0 = Sex, 1 = Age group, 2 = Student response, 3 = VECTOR, "
"4 = COORDINATE, "
" 5 = VALUE: ")
if column == "0":
self.list_records.sort(key=lambda x: x.sex)
elif column == "1":
self.list_records.sort(key=lambda x: x.age)
elif column == "2":
self.list_records.sort(key=lambda x: x.response)
elif column == "3":
self.list_records.sort(key=lambda x: x.vector)
elif column == "4":
self.list_records.sort(key=lambda x: x.coordinate)
elif column == "5":
self.list_records.sort(key=lambda x: int(x.value))
else:
print("Not a valid column number")
@staticmethod
def print_stars(num):
"""Static helper method to print out the number of stars to use in the histogram."""
for i in range(num):
print("*", end='')
print()
def print_chart(self, rarely, sometimes, often, always, dont):
"""Used with print_stars to create the formatted histogram and output it to console."""
print("How often do you wear your bicycle helmet while cycling?")
print("Rarely or never (", end='')
print(rarely, ") |", end='', sep='')
self.print_stars(rarely)
print("Sometimes (", end='')
print(sometimes, ") |", end='', sep='')
self.print_stars(sometimes)
print("Often (", end='')
print(often, ") |", end='', sep='')
self.print_stars(often)
print("Always (", end='')
print(always, ") |", end='', sep='')
self.print_stars(always)
print("Do not ride a bicycle (", end='')
print(dont, ") |", end='', sep='')
self.print_stars(dont)
def generate_histogram(self):
"""Retrieves input and generates histogram data based on user input. Loops through list of records and
increments the specific count when the response condition is satisfied. Uses print_chart to output the data."""
print("How you would like to visualize data chart?")
print(
"a: age and gender, b: aggregate female, c: aggregate male, d: 11 years, e: 13 years, "
"f: 15 years, g: all data")
response = input("Please enter an option found above: ")
response = response.lower()
count_rarely = 0
count_sometimes = 0
count_often = 0
count_always = 0
count_dont = 0
if response == "a":
gender = input("Male or female? Enter m: male, f: female: ")
gender = gender.lower()
if gender == "m":
gender = "Males"
elif gender == "f":
gender = "Females"
else:
print("Please enter a valid option. Back to main menu.")
return
age = input("Age range? Enter 11, 13, or 15: ")
if age == "11":
age = "11 years"
elif age == "13":
age = "13 years"
elif age == "15":
age = "15 years"
else:
print("Please enter a valid option. Back to main menu.")
return
for record in self.list_records:
if record.sex == gender and record.age == age and record.response == "Rarely or never":
count_rarely += 1
elif record.sex == gender and record.age == age and record.response == "Sometimes":
count_sometimes += 1
elif record.sex == gender and record.age == age and record.response == "Often":
count_often += 1
elif record.sex == gender and record.age == age and record.response == "Always":
count_always += 1
elif record.sex == gender and record.age == age and record.response == "Do not ride a bicycle":
count_dont += 1
print(gender, ", ", age, sep='')
self.print_chart(count_rarely, count_sometimes, count_often,
count_always, count_dont)
elif response == "b":
for record in self.list_records:
if record.sex == "Females" and record.response == "Rarely or never":
count_rarely += 1
elif record.sex == "Females" and record.response == "Sometimes":
count_sometimes += 1
elif record.sex == "Females" and record.response == "Often":
count_often += 1
elif record.sex == "Females" and record.response == "Always":
count_always += 1
elif record.sex == "Females" and record.response == "Do not ride a bicycle":
count_dont += 1
print("Females, all ages:")
self.print_chart(count_rarely, count_sometimes, count_often,
count_always, count_dont)
elif response == "c":
for record in self.list_records:
if record.sex == "Males" and record.response == "Rarely or never":
count_rarely += 1
elif record.sex == "Males" and record.response == "Sometimes":
count_sometimes += 1
elif record.sex == "Males" and record.response == "Often":
count_often += 1
elif record.sex == "Males" and record.response == "Always":
count_always += 1
elif record.sex == "Males" and record.response == "Do not ride a bicycle":
count_dont += 1
print("Males, all ages:")
self.print_chart(count_rarely, count_sometimes, count_often,
count_always, count_dont)
elif response == "d":
for record in self.list_records:
if record.age == "11 years" and record.response == "Rarely or never":
count_rarely += 1
elif record.age == "11 years" and record.response == "Sometimes":
count_sometimes += 1
elif record.age == "11 years" and record.response == "Often":
count_often += 1
elif record.age == "11 years" and record.response == "Always":
count_always += 1
elif record.age == "11 years" and record.response == "Do not ride a bicycle":
count_dont += 1
print("11 years old:")
self.print_chart(count_rarely, count_sometimes, count_often,
count_always, count_dont)
elif response == "e":
for record in self.list_records:
if record.age == "13 years" and record.response == "Rarely or never":
count_rarely += 1
elif record.age == "13 years" and record.response == "Sometimes":
count_sometimes += 1
elif record.age == "13 years" and record.response == "Often":
count_often += 1
elif record.age == "13 years" and record.response == "Always":
count_always += 1
elif record.age == "13 years" and record.response == "Do not ride a bicycle":
count_dont += 1
print("13 years old:")
self.print_chart(count_rarely, count_sometimes, count_often,
count_always, count_dont)
elif response == "f":
for record in self.list_records:
if record.age == "15 years" and record.response == "Rarely or never":
count_rarely += 1
elif record.age == "15 years" and record.response == "Sometimes":
count_sometimes += 1
elif record.age == "15 years" and record.response == "Often":
count_often += 1
elif record.age == "15 years" and record.response == "Always":
count_always += 1
elif record.age == "15 years" and record.response == "Do not ride a bicycle":
count_dont += 1
print("15 years old:")
self.print_chart(count_rarely, count_sometimes, count_often,
count_always, count_dont)
elif response == "g":
for record in self.list_records:
if record.response == "Rarely or never":
count_rarely += 1
elif record.response == "Sometimes":
count_sometimes += 1
elif record.response == "Often":
count_often += 1
elif record.response == "Always":
count_always += 1
elif record.response == "Do not ride a bicycle":
count_dont += 1
print("Males, Females, All ages:")
self.print_chart(count_rarely, count_sometimes, count_often,
count_always, count_dont)
else:
print("Please enter a valid option. Back to main menu.")
class MatchRater:
def __init__(self, str_id, df=None, rematch=False):
self.df = df
self.str_id = str_id
self.initialize()
self.ds = DataStore()
if rematch:
self.ds.rematch = True
def initialize(self):
self.root = Tk()
self.root.title("MatchRater 5000")
self.root.bind('<Return>', self.YES_onclick)
self.root.bind('<BackSpace>', self.NO_onclick)
self.root.bind('<space>', self.MAYBE_onclick)
self.root.columnconfigure(0, pad=100, minsize=450)
self.root.columnconfigure(1, pad=10, minsize=450)
self.root.rowconfigure(0, minsize=50)
def add_label(self, text, row, column, pady=10, padx=25, sticky=W):
lbl = Label(self.root, text=text, wraplength=350)
lbl.grid(row=row, column=column, pady=pady, padx=padx, sticky=sticky)
def gui_test(self):
self.add_label("Mentee name", 0, 0, padx=10, sticky=N)
self.add_label("Position", 1, 0)
self.add_label("University", 2, 0)
self.add_label(LOREM, 3, 0)
self.add_label("Suggestion 1", 4, 0)
self.add_label("Suggestion 2", 5, 0)
self.add_label("Suggestion 3", 6, 0)
self.add_label("Mentee name", 0, 1, padx=10, sticky=N)
self.add_label("Position", 1, 1)
self.add_label("Company", 2, 1)
self.add_label(LOREM, 3, 1)
self.add_label("Suggestion 1", 4, 1)
self.add_label("Suggestion 2", 5, 1)
self.add_label("Suggestion 3", 6, 1)
self.setup_bottom()
def setup_bottom(self, match_score=""):
lbl = Label(
self.root,
text=str(match_score) +
"\nWas this match good/plausible? E.g. could it have happened when we were matching?"
)
lbl.grid(row=7, column=0, columnspan=2, pady=(50, 5), sticky=S)
btn = Button(self.root, text="Yes", command=self.YES_onclick)
btn.grid(row=8, column=0, sticky=E, pady=(0, 50))
btn = Button(self.root, text="No", command=self.NO_onclick)
btn.grid(row=8, column=1, sticky=W, pady=(0, 50))
def show_match(self, mentor, mentee, score):
self.mentor = mentor
self.mentee = mentee
self.add_label(mentee.name, 0, 0, padx=10, sticky=N)
self.add_label(mentee.position, 1, 0)
self.add_label(mentee.university, 2, 0)
self.add_label(mentee.content, 3, 0)
"""
self.add_label(mentee.priorities[0], 4, 0)
self.add_label(mentee.priorities[1], 5, 0)
self.add_label(mentee.priorities[2], 6, 0)"""
self.add_label(mentor.name, 0, 1, padx=10, sticky=N)
self.add_label(mentor.position, 1, 1)
self.add_label(mentor.company, 2, 1)
self.add_label(mentor.content, 3, 1)
"""self.add_label(mentor.priorities[0], 4, 1)
self.add_label(mentor.priorities[1], 5, 1)
self.add_label(mentor.priorities[2], 6, 1)"""
self.setup_bottom(match_score=score)
def start(self):
self.root.mainloop()
def NO_onclick(self, event=None):
print("The match between " + self.mentee.name + " and " +
self.mentor.name + " was bad")
self.df = self.df.append(
{
"mentee_id": self.mentee.id,
"mentor_id": self.mentor.id,
"match_rating": 0
},
ignore_index=True)
for child in self.root.winfo_children():
child.destroy()
self.root.quit()
# self.root.destroy()
def MAYBE_onclick(self, event=None):
print("The match between " + self.mentee.name + " and " +
self.mentor.name + " was good!")
self.df = self.df.append(
{
"mentee_id": self.mentee.id,
"mentor_id": self.mentor.id,
"match_rating": -1
},
ignore_index=True)
for child in self.root.winfo_children():
child.destroy()
self.root.quit()
# self.root.destroy()
def YES_onclick(self, event=None):
print("The match between " + self.mentee.name + " and " +
self.mentor.name + " was good!")
self.df = self.df.append(
{
"mentee_id": self.mentee.id,
"mentor_id": self.mentor.id,
"match_rating": 1
},
ignore_index=True)
for child in self.root.winfo_children():
child.destroy()
self.root.quit()
# self.root.destroy()
# TODO: Fix issues with saving (probably caused by issue in YES/NO_onclick)
def export_csv(self):
self.ds.save_to_disk(df=self.df)
self.ds.append_matching_info(df=self.df, str_id=self.str_id)
def kill(self):
self.root.destroy()
def get_cache(self):
cached_matches = {}
duplicated = {}
for file in os.listdir("data/match_rating"):
results = self.ds.load_rating_file(file)
for i, result in results.iterrows():
dict_key = str(result["mentor_id"]) + str(result["mentee_id"])
# If the key already exists
if dict_key in cached_matches and dict_key not in duplicated:
if result["match_rating"] != cached_matches[dict_key]:
duplicated[dict_key] = result["match_rating"]
del (cached_matches[dict_key])
continue
# If the key does not exist
cached_matches[dict_key] = result["match_rating"]
return cached_matches
def rate_match(self, mentor, mentee, rating):
self.df = self.df.append(
{
"mentee_id": mentee.id,
"mentor_id": mentor.id,
"match_rating": rating
},
ignore_index=True)
def __init__(self):
tok = "8051bf89-e115-4147-8e5a-ff9d6f39f0d7"
url = "hermes.wha.la"
value = {"Command": "INIT", "Token": tok}
headers = {
'Content-type': 'application/json',
'Accept': 'application/json',
}
jvalue = json.dumps(value)
conn = httplib.HTTPConnection(url, 80)
conn.request('POST', '/api/hermes', jvalue, headers)
response = conn.getresponse()
ret = json.loads(
str((response.status, response.reason, response.read())[2]))
conn.close()
DS = DataStore(
ret["ServerState"]["ServerTiers"]["DB"]["ServerStartTurnTime"])
ctrl = controller()
# myFile = open('output.txt', 'w')
# myFile.write(str(ret))
# myFile.close()
looptime = ret['ServerState']['ServerTiers']['DB'][
'ServerStartTurnTime']
coef = (ret["ServerState"]["CostPerServer"] /
ret["ServerState"]["ProfitConstant"])
WebRefresh = ret["ServerState"]["ServerTiers"]["WEB"][
"ServerStartTurnTime"]
JavaRefresh = ret["ServerState"]["ServerTiers"]["JAVA"][
"ServerStartTurnTime"]
DBRefresh = ret["ServerState"]["ServerTiers"]["DB"][
"ServerStartTurnTime"]
DS.setCoef(coef)
infra = False
p = None
research = None
didGrid = False
progression = [None, "GRID", "GREEN", None]
#while ret['ServerState']['TurnNo'] < 10080:
while True:
x = 0
value = {"Command": "PLAY", "Token": tok}
headers = {
'Content-type': 'application/json',
'Accept': 'application/json',
}
jvalue = json.dumps(value)
conn = httplib.HTTPConnection(url, 80)
conn.request('POST', '/api/hermes', jvalue, headers)
response = conn.getresponse()
ret = json.loads(
str((response.status, response.reason, response.read())[2]))
turnnumber = ret['ServerState']['TurnNo']
#Get demand from server
demand = [
ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['NA']
['NoOfTransactionsInput']
]
demand.append(ret['ServerState']['ServerTiers']['WEB']
['ServerRegions']['EU']['NoOfTransactionsInput'])
demand.append(ret['ServerState']['ServerTiers']['WEB']
['ServerRegions']['AP']['NoOfTransactionsInput'])
config = [
ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['NA']
['NodeCount']
]
config.append(ret['ServerState']['ServerTiers']['WEB']
['ServerRegions']['EU']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['WEB']
['ServerRegions']['AP']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['JAVA']
['ServerRegions']['NA']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['JAVA']
['ServerRegions']['EU']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['JAVA']
['ServerRegions']['AP']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['DB']
['ServerRegions']['NA']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['DB']
['ServerRegions']['EU']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['DB']
['ServerRegions']['AP']['NodeCount'])
# if turnnumber % (DBRefresh*2):
# DS.resetDemand(demand)
# else:
# DS.avgDemand(demand)
DS.runningDemand(demand)
DS.setConfig(config)
coef = (ret["ServerState"]["CostPerServer"] /
ret["ServerState"]["ProfitConstant"])
DS.setCoef(coef)
conn.close()
lastProfit = ret["ServerState"]["ProfitEarned"]
GridCost = ret["ServerState"]["ResearchUpgradeLevels"][1][
"UpgradeCost"]
GridTurns = ret["ServerState"]["ResearchUpgradeLevels"][1][
"NoOfTurnsRequired"]
GridTotalCost = GridTurns * GridCost
#init research when its swaggin to do so
if ret['ServerState']['TurnNo'] <= 2000 and ret["ServerState"][
"ProfitAccumulated"] >= GridTotalCost / 8 and GridCost < (
lastProfit - (lastProfit / 3)):
didGrid = True
try:
if ret["ServerState"]["ResearchUpgradeState"][
"GRID"] == -1:
#research = "GREEN"
pass
except:
research = "GRID"
pass
#p = research
#Calculate free space
#AVERAGE CAPACITY
# capacity = [ ( ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2]
# capacity.append(( ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2)
# capacity.append(( ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2)
#97%
capacity = [
(ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']
['CapactityLevels'][0]['UpperLimit'] +
ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']
['CapactityLevels'][1]['UpperLimit'] +
ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']
['CapactityLevels'][0]['UpperLimit']) / 3
]
capacity.append(
(ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']
['CapactityLevels'][0]['UpperLimit'] +
ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']
['CapactityLevels'][1]['UpperLimit'] +
ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']
['CapactityLevels'][0]['UpperLimit']) / 3)
capacity.append(
(ret['ServerState']['ServerTiers']['DB']['ServerPerformance']
['CapactityLevels'][0]['UpperLimit'] +
ret['ServerState']['ServerTiers']['DB']['ServerPerformance']
['CapactityLevels'][1]['UpperLimit'] +
ret['ServerState']['ServerTiers']['DB']['ServerPerformance']
['CapactityLevels'][0]['UpperLimit']) / 3)
#93%
# capacity = [ ( ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']+ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 3]
# capacity.append(( ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 3)
# capacity.append(( ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']) / 3)
#100% CAPACITY
# capacity = [ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit']]
# capacity.append(ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'])
# capacity.append(ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'])
# #90% CAPACITY
# capacity = [ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']]
# capacity.append(ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'])
# capacity.append(ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'])
DS.setCapacity(capacity)
webchanges = [0, 0, 0, 0, 0, 0, 0, 0, 0]
javachanges = [0, 0, 0, 0, 0, 0, 0, 0, 0]
dbchanges = [0, 0, 0, 0, 0, 0, 0, 0, 0]
if turnnumber % WebRefresh == 0:
webchanges = ctrl.calcWeb(DS, WebRefresh * 1.5)
if turnnumber % JavaRefresh == 0:
javachanges = ctrl.calcJava(DS, JavaRefresh * 1.5)
if turnnumber % DBRefresh == 0:
dbchanges = ctrl.calcDB(DS, DBRefresh * 1.5)
#create 'changes' to know what servers to bring up/down
changes = []
changes.append(webchanges[0])
changes.append(webchanges[1])
changes.append(webchanges[2])
changes.append(javachanges[3])
changes.append(javachanges[4])
changes.append(javachanges[5])
changes.append(dbchanges[6])
changes.append(dbchanges[7])
changes.append(dbchanges[8])
jsonchange = {
"Servers": {
"WEB": {
"ServerRegions": {
"AP": {
"NodeCount": changes[2]
},
"EU": {
"NodeCount": changes[1]
},
"NA": {
"NodeCount": changes[0]
}
}
},
"JAVA": {
"ServerRegions": {
"NA": {
"NodeCount": changes[3]
},
"EU": {
"NodeCount": changes[4]
},
"AP": {
"NodeCount": changes[5]
}
}
},
"DB": {
"ServerRegions": {
"NA": {
"NodeCount": changes[6]
},
"EU": {
"NodeCount": changes[7]
},
"AP": {
"NodeCount": changes[8]
}
}
}
},
"UpgradeInfraStructure": infra,
"UpgradeToResearch": research
}
if research != None:
research = None
value = {
"Command": "CHNG",
"Token": tok,
"ChangeRequest": jsonchange
}
headers = {
'Content-type': 'application/json',
'Accept': 'application/json',
}
jvalue = json.dumps(value)
conn = httplib.HTTPConnection(url, 80)
conn.request('POST', '/api/hermes', jvalue, headers)
response = conn.getresponse()
ret2 = json.loads(
str((response.status, response.reason, response.read())[2]))
conn.close()
#print Stuff
print 'Turn: ' + str(ret['ServerState']['TurnNo'])
print 'Total Profit: $' + str(
ret["ServerState"]["ProfitAccumulated"])
# print "WEB capacity: " + str(capacity[0])
# print "JAVA capacity: " + str(capacity[1])
# print "DB capacity: " + str(capacity[2])
# print "ServerCost: " + str(ret["ServerState"]["CostPerServer"])
#print didGrid
#if didGrid:
try:
inf = str(
ret["ServerState"]["InfraStructureUpgradeState"]["Value"])
if inf >= 0:
#print "INFRA value: " + inf
pass
except:
pass
try:
grid = str(ret["ServerState"]["ResearchUpgradeState"]["GRID"])
if grid != "-1":
print "---Researching: " + "GRID" + "---\nTurns Left: " + grid
if int(grid) <= 1441 and int(grid) >= 1430:
#infra = True
pass
else:
infra = False
else:
print "GRID UPGRADE COMPLETE"
except:
pass
try:
green = str(
ret["ServerState"]["ResearchUpgradeState"]["GREEN"])
if green != "-1":
print "---Researching: " + "GREEN" + "---\nTurns Left: " + green
else:
print "GREEN UPGRADE COMPLETE"
except:
pass
print demand
print ' ' + str(config[0]) + ' ' + str(
config[1]) + ' ' + str(config[2]) + ' ' + '\n ' + str(
config[3]) + ' ' + str(config[4]) + ' ' + str(
config[5]) + ' ' + '\n ' + str(
config[6]) + ' ' + str(
config[7]) + ' ' + str(config[8])
print ''
conn.close()
'''
import logging
#logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger()
from Probe import Probe
from PPPParam import PPPParam
from ResGraph import ResGraph
from ProbeController import ProbeController
from DataStore import DataStore
# ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ----
data_store = DataStore()
probe1 = Probe('ping', "ping -c 1 localhost")
probe2 = Probe('curl', "curl -s http://localhost/")
probes = [probe1, probe2]
ppp_params = [PPPParam(60), PPPParam(900), PPPParam()]
for probe in probes:
data_store.add(probe)
data_store.commit()
res_graph = ResGraph(probes, ppp_params)
probe_controller = ProbeController(probes, res_graph, data_store)
probe_controller.start()
SC_list = SeparateCollects(path_to_directory_of_ast_files, AGO.file_name[0:12])
print('Separated collect files for two collects: ', SC_list.collect_files)
# Example storing frame stacks, labels, and mean object
# centroids in a .hdf5 file. DataStore compresses the
# .hdf5 file to reduce the required disk space.
name_of_labels_csv =\
'dataset_labels.csv'
path_to_directory_of_fits_files =\
'/home/jermws/Sample Raven Images/FITS'
name_of_hdf5_file_for_storage =\
'labeled_datasets.hdf5'
path_to_directory_for_storage =\
'/home/jermws/PycharmProjects/Machine_Learning_Updated/'
DS = DataStore(name_of_labels_csv, path_to_directory_of_fits_files,
path_to_directory_of_ast_files, name_of_hdf5_file_for_storage,
path_to_directory_for_storage)
# Save the list of data set names in the .hdf5 file
# so they can be recalled when it comes time to read
# the data back in.
with open('dataset_names.csv', 'w') as f:
w = csv.writer(f)
for row in DS.dataset_names:
w.writerow([row])
# Read in the data set names so they can be used to
# label the chip stacks.
def __init__(self):
"""Default Constructor to initialize data/persistence layer and fetch records from csv."""
self.data_store = DataStore()
self.list_records = self.data_store.get_all_records()
class SpiderMan:
def __init__(self):
self.data_store = DataStore()
self.url_manager = UrlManager(self.data_store)
self.strategy_container = ParserStrategyContainer()
self.downloader = HtmlDownloader()
def start(self):
self.register_station()
while self.has_task():
try:
self.handle_station()
self.handle_content()
except Exception as e:
print(e)
def has_task(self):
station_count = self.url_manager.get_stations_size()
contents_count = self.url_manager.get_contents_size()
all_count = station_count + contents_count
return all_count > 0
def register_station(self):
strategys = self.strategy_container.get_strategys()
for key, value in strategys.items():
self.data_store.add_station_url(key, value.get_url())
def handle_station(self):
data = self.url_manager.get_station_url()
if data is None:
return
strategy = self.strategy_container.get_srategy(data['station'])
# get all urls
html = self.downloader.download(data['url'])
#for test
#fin = open("test", "w", encoding="utf-8")
#fin.write(html)
#fin = open("test", "r", encoding="utf-8")
#html = fin.read();
content_urls = strategy.parser_station(data['url'], html)
# store urls to mysql
perform_count = 0
for content_url in content_urls:
disease = content_url['disease']
tmp_url = content_url['url']
perform_count += self.url_manager.add_content_url(
tmp_url, disease, data['station'])
#updata sation state
state = 1
if len(content_urls) > perform_count or len(content_urls) == 0:
state = -1
self.data_store.updata_staion_state(state, data['station'])
def handle_content(self):
data = self.url_manager.get_content_url()
strategy = self.strategy_container.get_srategy(data['station'])
# get page contents
html = self.downloader.download(data['url'])
contents = strategy.parser_content(html)
# store contents to mysql
state = 0
code = 1
contents['knowledge_id'] = data['knowledge_id']
if self.data_store.add_content(contents) < 0:
state = -1
#update state
self.data_store.update_page_state(code, data['knowledge_id'])
def __init__(self):
value = {
"Command": "INIT",
"Token": "8051bf89-e115-4147-8e5a-ff9d6f39f0d7"
}
headers = {
'Content-type': 'application/json',
'Accept': 'application/json',
}
jvalue = json.dumps(value)
conn = httplib.HTTPConnection('107.20.243.77', 80)
conn.request('POST', '/api/hermes', jvalue, headers)
response = conn.getresponse()
ret = json.loads(
str((response.status, response.reason, response.read())[2]))
conn.close()
DS = DataStore()
ctrl = controller()
myFile = open('output.txt', 'w')
myFile.write(str(ret))
myFile.close()
looptime = ret['ServerState']['ServerTiers']['DB'][
'ServerStartTurnTime']
coef = (ret["ServerState"]["CostPerServer"] /
ret["ServerState"]["ProfitConstant"])
DS.setCoef(coef)
infra = False
p = None
research = None
didGrid = False
progression = [None, "GRID", "GREEN", None]
#while ret['ServerState']['TurnNo'] < 10080:
while True:
x = 0
while x <= looptime - 1:
value = {
"Command": "PLAY",
"Token": "8051bf89-e115-4147-8e5a-ff9d6f39f0d7"
}
headers = {
'Content-type': 'application/json',
'Accept': 'application/json',
}
jvalue = json.dumps(value)
conn = httplib.HTTPConnection('107.20.243.77', 80)
conn.request('POST', '/api/hermes', jvalue, headers)
response = conn.getresponse()
ret = json.loads(
str((response.status, response.reason,
response.read())[2]))
#Get demand from server
demand = [
ret['ServerState']['ServerTiers']['DB']['ServerRegions']
['NA']['NoOfTransactionsInput']
]
demand.append(ret['ServerState']['ServerTiers']['DB']
['ServerRegions']['EU']['NoOfTransactionsInput'])
demand.append(ret['ServerState']['ServerTiers']['DB']
['ServerRegions']['AP']['NoOfTransactionsInput'])
config = [
ret['ServerState']['ServerTiers']['WEB']['ServerRegions']
['NA']['NodeCount']
]
config.append(ret['ServerState']['ServerTiers']['WEB']
['ServerRegions']['EU']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['WEB']
['ServerRegions']['AP']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['JAVA']
['ServerRegions']['NA']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['JAVA']
['ServerRegions']['EU']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['JAVA']
['ServerRegions']['AP']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['DB']
['ServerRegions']['NA']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['DB']
['ServerRegions']['EU']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['DB']
['ServerRegions']['AP']['NodeCount'])
DS.avgDemand(demand)
DS.setConfig(config)
conn.close()
x += 1
lastProfit = ret["ServerState"]["ProfitEarned"]
GridCost = ret["ServerState"]["ResearchUpgradeLevels"][1][
"UpgradeCost"]
GridTurns = ret["ServerState"]["ResearchUpgradeLevels"][1][
"NoOfTurnsRequired"]
GridTotalCost = GridTurns * GridCost
if ret['ServerState']['TurnNo'] <= 9000 and ret["ServerState"][
"ProfitAccumulated"] >= GridTotalCost / 10 and GridCost < (
lastProfit - (lastProfit / 3)):
didGrid = True
try:
if ret["ServerState"]["ResearchUpgradeState"][
"GRID"] == -1:
#research = "GREEN"
pass
except:
research = "GRID"
#p = research
#Calculate free space
#AVERAGE CAPACITY
# capacity = [ ( ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2]
# capacity.append(( ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2)
# capacity.append(( ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 2)
#97%
capacity = [
(ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']
['CapactityLevels'][0]['UpperLimit'] +
ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']
['CapactityLevels'][1]['UpperLimit'] +
ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']
['CapactityLevels'][0]['UpperLimit']) / 3
]
capacity.append(
(ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']
['CapactityLevels'][0]['UpperLimit'] +
ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']
['CapactityLevels'][1]['UpperLimit'] +
ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']
['CapactityLevels'][0]['UpperLimit']) / 3)
capacity.append(
(ret['ServerState']['ServerTiers']['DB']['ServerPerformance']
['CapactityLevels'][0]['UpperLimit'] +
ret['ServerState']['ServerTiers']['DB']['ServerPerformance']
['CapactityLevels'][1]['UpperLimit'] +
ret['ServerState']['ServerTiers']['DB']['ServerPerformance']
['CapactityLevels'][0]['UpperLimit']) / 3)
#93%
# capacity = [ ( ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']+ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 3]
# capacity.append(( ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] ) / 3)
# capacity.append(( ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'] + ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']) / 3)
#100% CAPACITY
# capacity = [ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit']]
# capacity.append(ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'])
# capacity.append(ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][0]['UpperLimit'])
# #90% CAPACITY
# capacity = [ ret['ServerState']['ServerTiers']['WEB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit']]
# capacity.append(ret['ServerState']['ServerTiers']['JAVA']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'])
# capacity.append(ret['ServerState']['ServerTiers']['DB']['ServerPerformance']['CapactityLevels'][1]['UpperLimit'])
DS.setCapacity(capacity)
changes = ctrl.calc(DS)
jsonchange = {
"Servers": {
"WEB": {
"ServerRegions": {
"AP": {
"NodeCount": changes[2]
},
"EU": {
"NodeCount": changes[1]
},
"NA": {
"NodeCount": changes[0]
}
}
},
"JAVA": {
"ServerRegions": {
"NA": {
"NodeCount": changes[3]
},
"EU": {
"NodeCount": changes[4]
},
"AP": {
"NodeCount": changes[5]
}
}
},
"DB": {
"ServerRegions": {
"NA": {
"NodeCount": changes[6]
},
"EU": {
"NodeCount": changes[7]
},
"AP": {
"NodeCount": changes[8]
}
}
}
},
"UpgradeInfraStructure": infra,
"UpgradeToResearch": research
}
if research != None:
research = None
value = {
"Command": "CHNG",
"Token": "8051bf89-e115-4147-8e5a-ff9d6f39f0d7",
"ChangeRequest": jsonchange
}
headers = {
'Content-type': 'application/json',
'Accept': 'application/json',
}
jvalue = json.dumps(value)
conn = httplib.HTTPConnection('107.20.243.77', 80)
conn.request('POST', '/api/hermes', jvalue, headers)
response = conn.getresponse()
ret = json.loads(
str((response.status, response.reason, response.read())[2]))
# if research != None:
# research = None
# jsonchange = {"Servers":{
# "WEB":{"UpgradeToResearch": "Green"}}}
# value = {
# "Command": "CHNG",
# "Token": "8051bf89-e115-4147-8e5a-ff9d6f39f0d7",
# #"Token": "7440b0b0-c5a2-4ab3-bdc3-8935865bb9d1",
# "ChangeRequest": jsonchange
# }
# headers = {'Content-type': 'application/json','Accept': 'application/json',}
# jvalue = json.dumps(value)
# conn = httplib.HTTPConnection('107.20.243.77', 80)
# #conn = httplib.HTTPConnection('uat.hermes.wha.la', 80)
# conn.request('POST', '/api/hermes', jvalue, headers)
# response = conn.getresponse()
# ret = json.loads(str((response.status, response.reason, response.read())[2]))
#play
value = {
"Command": "PLAY",
"Token": "8051bf89-e115-4147-8e5a-ff9d6f39f0d7"
}
headers = {
'Content-type': 'application/json',
'Accept': 'application/json',
}
jvalue = json.dumps(value)
conn = httplib.HTTPConnection('107.20.243.77', 80)
conn.request('POST', '/api/hermes', jvalue, headers)
response = conn.getresponse()
ret = json.loads(
str((response.status, response.reason, response.read())[2]))
#Get demand from server
demand = [
ret['ServerState']['ServerTiers']['DB']['ServerRegions']['NA']
['NoOfTransactionsInput']
]
demand.append(ret['ServerState']['ServerTiers']['DB']
['ServerRegions']['EU']['NoOfTransactionsInput'])
demand.append(ret['ServerState']['ServerTiers']['DB']
['ServerRegions']['AP']['NoOfTransactionsInput'])
config = [
ret['ServerState']['ServerTiers']['WEB']['ServerRegions']['NA']
['NodeCount']
]
config.append(ret['ServerState']['ServerTiers']['WEB']
['ServerRegions']['EU']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['WEB']
['ServerRegions']['AP']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['JAVA']
['ServerRegions']['NA']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['JAVA']
['ServerRegions']['EU']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['JAVA']
['ServerRegions']['AP']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['DB']
['ServerRegions']['NA']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['DB']
['ServerRegions']['EU']['NodeCount'])
config.append(ret['ServerState']['ServerTiers']['DB']
['ServerRegions']['AP']['NodeCount'])
DS.resetDemand(demand)
DS.setConfig(config)
coef = (ret["ServerState"]["CostPerServer"] /
ret["ServerState"]["ProfitConstant"])
DS.setCoef(coef)
# f = open('myfile.txt','w')
# f.write(str(ret)) # python will convert \n to os.linesep
# f.close()
conn.close()
print 'Turn: ' + str(ret['ServerState']['TurnNo'])
print "WEB capacity: " + str(capacity[0])
print "JAVA capacity: " + str(capacity[1])
print "DB capacity: " + str(capacity[2])
print "ServerCost: " + str(ret["ServerState"]["CostPerServer"])
#print didGrid
#if didGrid:
try:
inf = str(
ret["ServerState"]["InfraStructureUpgradeState"]["Value"])
if inf >= 0:
print "INFRA value: " + inf
except:
pass
try:
grid = str(ret["ServerState"]["ResearchUpgradeState"]["GRID"])
if grid != "-1":
print "---Researching: " + "GRID" + "---\nTurns Left: " + grid
if int(grid) <= 1441 and int(grid) >= 1430:
#infra = True
pass
else:
infra = False
else:
print "GRID UPGRADE COMPLETE"
except:
pass
try:
green = str(
ret["ServerState"]["ResearchUpgradeState"]["GREEN"])
if green != "-1":
print "---Researching: " + "GREEN" + "---\nTurns Left: " + green
else:
print "GREEN UPGRADE COMPLETE"
except:
pass
print demand
print ' ' + str(config[0]) + ' ' + str(
config[1]) + ' ' + str(config[2]) + ' ' + '\n ' + str(
config[3]) + ' ' + str(config[4]) + ' ' + str(
config[5]) + ' ' + '\n ' + str(
config[6]) + ' ' + str(
config[7]) + ' ' + str(config[8])
print ''
conn.close()
import pandas as pd
import numpy as np
from datetime import datetime
import _pickle as pk
import json
from DataStore import DataStore
# from FHMM import FHMM
# from HMM import HMM, HMM_MAD
from Preprocessing import Appliance, train_test_split, create_matrix
#with open('aws_keys.json') as f:
# data = json.load(f)
# ACCESS_KEY = data['access-key']
# SECRET_ACCESS_KEY = data['secret-access-key']
DStore = DataStore('ngalvbucket1', 'house_1')
all_channels = [1, 12, 5, 3, 10, 6, 9, 43, 7, 8]
select_channels = [12, 5, 3, 10, 6, 9, 43, 8]
# select_channels = [12, 5, 6]
DStore.create_store(all_channels)
#top_10 = DStore.select_top_k(10,'2013-08-01','2013-09-01')
combined = DStore.create_combined_df('2013-06-01 00:00:00',
'2013-10-31 23:59:59',
select_channels=select_channels,
freq='1Min')
with open('combined.pkl', 'wb') as f:
pk.dump(combined, f)
class ReportGenerator(object):
"""Assists with generating histogram reports
Bundles all of the Report object generation things in one place and adds
convenience methods for generation
Attributes:
config(ReportConfig): A ReportConfig object that the generator passes to each
generated Report
programs(list(string)): A list of programs (e.g. ENCM) to process.
whitelist(dictionary): A dictionary including lists of the only things that should
be processed. Options can include, but are not limited to, lists of:
* Courses
* Indicators
* Assessments
As long as the whitelist key matches closely to the column name in
the indicator lookup table, it should be fine
ds(DataStore): A DataStore object
TODO:
* Implement a blacklisting feature
"""
def __init__(
self,
config,
programs=None,
cohorts=None,
whitelist=None,
ds=None,
indicators_loc=None,
grades_loc=None,
histograms_loc=None,
):
"""Object Initialization
Args:
config(ReportConfig): A ReportConfig object
programs(string or list(string)): A list of programs to generate indicators for.
Defaults to using all programs. If only one string is passed in, it will
get put into a size-one list.
cohorts(int or list(int)): A list of cohorts to pull data from. Defaults to
using all cohorts. If only one cohort is passed in, it will get put into
a size-one list.
whitelist(dictionary): A dictionary of lists, keyed by the only stuff
to parse (e.g. 'course', 'indicator', etc.) and filled with the
specific values to uniquely parse. Defaults to no whitelist. The lists
in the dictionaries can be partial (i.e. if you pass 'KB' as part of the
'indicator' whitelist, it will pull all 'KB' indicators). If the
lists contain only one string, that string gets put into a
size-one list.
ds(DataStore): A DataStore object. Defaults to generating one based on the
whitelist entries for 'programs'
indicators_loc(string): The location of the indicator sheets. Defaults to
searching an "Indicators" folder in the directory above the project
(see ReportConfig for more info)
grades_loc(string): The location of the grades sheets. Defaults to searching
a "Grades" folder in the directory above the project (see
ReportConfig for more info)
histograms_loc(string): The location to store histograms. Defaults to using
a "Histograms" folder in the directory above the project (see
ReportConfig for more info)
"""
logging.info("Start of AutoGenerator initialization")
self.config = config
# logging.info("Initializing whitelist")
self.whitelist = whitelist
# Ensure that all whitelist entries are lists (if it exists, that is)
if self.whitelist:
logging.debug("Checking whitelist for list validity")
for entry in self.whitelist.keys():
self.whitelist[entry] = _check_list(self.whitelist[entry])
# if type(self.whitelist[entry]) is not type(list()):
# logging.debug("Whitelist entry %s is not a list. Converting to a one-size list", entry)
# # Change to one-size list
# self.whitelist[entry] = [self.whitelist[entry]]
# logging.info("Initializing cohorts list")
self.cohorts = cohorts
# Ensure that all cohorts entries are lists (if it exists, that is)
if self.cohorts:
logging.debug("Checking cohorts variable for list validity")
self.cohorts = _check_list(self.cohorts)
# logging.info("Initializing program list")
self.programs = programs
# Ensure list validity of programs
if self.programs:
logging.debug("Checking programs for list validity")
self.programs = _check_list(self.programs)
# Use all programs if none were provided
if not self.programs:
logging.debug(
"Programs not passed as paramater. Using list of all programs")
self.programs = globals.all_programs
# # Same check as whitelist - ensure that self.programs is a list
# if type(self.programs) is not type(list()):
# logging.debug("Programs was not passed in list format. Converting to a one-size list")
# self.programs = [self.programs]
# If any of the file location parameters were passed in, overwrite what ReportConfig has
if indicators_loc:
self.config.indicators_loc = indicators_loc
if grades_loc:
self.config.grades_loc = grades_loc
if histograms_loc:
self.config.histograms_loc = histograms_loc
logging.debug("Indicators location is %s", self.config.indicators_loc)
logging.debug("Grades location is %s", self.config.grades_loc)
logging.debug("Histograms location is %s", self.config.histograms_loc)
# Check to see if a DataStore was passed to init, create one if not
if not ds:
logging.debug(
"No DataStore object was passed to init; creating one now")
self.ds = DataStore(programs=self.programs,
indicators_loc=self.config.indicators_loc,
grades_loc=self.config.grades_loc)
# Make sure that the histograms folder exists
logging.info(
"Setting up output directories (Missing Data & Histograms)")
os.makedirs(os.path.dirname(__file__) + '/../Missing Data',
exist_ok=True)
os.makedirs(self.config.histograms_loc, exist_ok=True)
logging.info("ReportGenerator initialization done!")
def _parse_row(self, row, program):
"""Turn a row of a Pandas DataFrame into a dictionary and bin list
The data stored in the master indicator spreadsheets needs to be cleaned up
properly for a Report object to use it. This function does that based on things
in the ReportConfig. Program gets passed to the function because that is currently
not stored in the indicators spreadsheet and needs to end up on the Report object
somehow. This seemed like the most reasonable place to do it.
ReportConfig.header_attribs determines what data gets pulled from the
spreadsheet. For a single entry in header_attribs, multiple values
could get pulled and concatenated from the spreadsheet row. For example,
writing just 'Course' in the header_attribs would join the columns
'Course #' and 'Course Description' using a ' - ' character
Args:
row(pd.Series): The row of a Pandas DataFrame (so a Pandas Series) to pull data from
program(string): The program that the indicator file row belongs to
Returns:
dict: A dictionary containing the indicator information, keyed using instructions
list(float): The bin ranges that have been converted from a comma-separated string
to a list of floats. 0 gets appended to the front for NumPy histogram
None, None: Returns 2 None values if non-number bins are found in the row
"""
# logging.info("Parsing a row from a Pandas DataFrame")
# Handle the bins first since those are easy
try:
bins = [float(x) for x in row['Bins'].split(',')]
except Exception:
logging.warning(
"ERROR: Non-number bins encountered in a lookup table")
return None, None
logging.debug("Bins parsed as:\t%s", ', '.join(str(x) for x in bins))
# Convert the comma-separated string into dictionary keys
# logging.info("Creating a dictionary of indicator information")
indicator_dict = {
i: ""
for i in
[x.strip() for x in self.config.header_attribs.split(',')]
}
# logging.info("Filling the dictionary with information from the row")
for key in indicator_dict.keys():
# Look for all occurrences where the key is found in the row's columns
# and store that information in a list
occurrences = [str(row[i]) for i in row.index if key in i]
# Glue all collected data together with ' - ' characters
# logging.debug("Indicator entry %s being filled with info from lookup table columns %s",
# key,
# ', '.join(occurrences)
# )
indicator_dict[key] = ' - '.join(occurrences)
# logging.debug("Entry [%s]: %s", key, indicator_dict[key])
# If Program is in indicator_dict, add the program parameter to indicator_dict
if 'Program' in indicator_dict.keys():
indicator_dict['Program'] = program
# logging.info("Returning indicator dictionary and bins from this row")
return indicator_dict, bins
def start_autogenerate(self):
"""Begin autogeneration of reports
The general procedure goes as follows:
* Iterate across the stored indicator lookup tables:
* Query the indicator table using the whitelist
* Iterate through the query:
* Get indicator data from the row
* Search for grades for the assessment tool
* Iterate through each file found and save a histogram
TODO:
* Decouple the method
* Change the histogram file naming conventions to be shorter
* Allow customization of the file save name through ReportConfig
"""
logging.info("Beginning report autogeneration")
logging.debug("Autogenerator set up to use programs %s",
', '.join(self.programs))
# Iterate across the list of programs
for program in self.programs:
logging.info("Generating reports for program %s", program)
# logging.info("Getting a query list from the program's indicator lookup table")
# Query the indicators DataFrame
query = self.ds.query_indicators(program=program,
dict_of_queries=self.whitelist)
# Set up a file to store missing data in
# logging.info("Starting a file to save missing data")
missing_data = open(
"../Missing Data/{} missing data.txt".format(program), "w+")
# Iterate across each indicator (each row of the query)
# logging.info("Beginning row iteration...")
for rownumber, row in query.iterrows():
# Skip this row if the "Assessed" column is set to any form of "No" (also check if it's there)
if 'Assessed' in row and row['Assessed'].lower() == 'no':
# print("Found a non-assessed indicator")
continue
# Skip this row if no bins are defined
if row['Bins'] in [np.nan, None]:
logging.warning(
"No bins (and likely no data) found for {} {} {} {}, skipping row"
.format(row['Indicator #'], row['Level'],
row['Course #'], row['Method of Assessment']))
logging.warning("Missing binning stored in separate file")
# Log the missing data entry
missing_data.write(
"Missing bin ranges for {a} {b} ({c}-{d})\n".format(
a=row["Course #"],
b=row["Method of Assessment"],
c=row['Indicator #'],
d=row['Level']))
continue
else:
logging.debug("Processing data for {} {} {} {}".format(
row['Indicator #'], row['Level'], row['Course #'],
row['Method of Assessment']))
#------------------------------------------------------------------------------
# Obtain the necessary indicator data and bins from the row. header_attribs in
# ReportConfig determine what gets pulled and what doesn't. These are also
# JSON-loadable. Program gets added to indicator data later if required.
#------------------------------------------------------------------------------
indicator_data, bins = self._parse_row(row, program)
# If no bins were parsed, a histogram cannot be generated. Skip this indicator/row
if not bins:
logging.warning(
"ERROR: No useable bins for {} {} {} {}, skipping row".
format(row['Indicator #'], row['Level'],
row['Course #'], row['Method of Assessment']))
# Log the missing bins
missing_data.write(
"No useable bins for {a} {b} ({c}-{d})\n".format(
a=row["Course #"],
b=row["Method of Assessment"],
c=row['Indicator #'],
d=row['Level']))
continue
logging.debug("Indicator data obtained from the row: %s",
str(indicator_data))
#-----------------------------------------------------------------------------
# Search for grade files. Store all occurrences of the assessment data so
# that a histogram can be generated for each data set.
#
# Generates a defaultdict like the following example:
# {
# "ENCM": [],
# "Core": ["ENGI 1040 Circuits Grade - Core - Custom column names.xlsx"],
# "ECE": ["ENGI 1040 Circuits Grade - ECE - Custom column names.xlsx"]
# }
# These are stored in lists because the original intention was to allow
# multiple assessment files for the same indicator.
#-----------------------------------------------------------------------------
# Use the config parameter to get the backup subdirectories
search_list = [program] + [
x.strip() for x in self.config.grade_backup_dirs.split(',')
]
logging.debug("Beginning search for grade files. Priority: %s",
', '.join(search_list))
found_grade_files = grades_org.directory_search(
course=row['Course #'],
assessment=row['Method of Assessment'],
main_dir=self.config.grades_loc,
subdirs=search_list)
# # Check the unique course table to see if the course has a unique term offering
# term_offered = None
# for _, unique_course in self.ds.unique_courses.iterrows():
# if row['Course #'] == unique_course['Course #']:
# term_offered = unique_course['Term Offered']
# break
# Run histogram generation and saving
self.gen_and_save_histograms(dirs_and_files=found_grade_files,
dir_priorities=search_list,
indicator_data=indicator_data,
bins=bins,
row=row,
rownum=rownumber,
program=program,
missing_data=missing_data)
# # If the lists are empty, add data entry to missing data file
# if not any(found_grade_files[k] for k in found_grade_files.keys()):
# missing_thing = "{a} {b} ({c}-{d})".format(
# a=row["Course #"],
# b=row["Method of Assessment"],
# c=row['Indicator #'],
# d=row['Level']
# )
# logging.warning("No data found for %s", missing_thing)
# missing_data.write("Missing data for {}\n".format(missing_thing))
# #-----------------------------------------------------------------------------
# # Set up the found_grade_files dictionary in a way that the program will
# # recognize as missing data and generate a blank histogram for
# #-----------------------------------------------------------------------------
# found_grade_files = {program: [None]}
# else:
# logging.debug("Found files in %s", ', '.join(found_grade_files.keys()))
# #-----------------------------------------------------------------------------
# # Iterate across the list of grade files found and separately generate a
# # histogram for each file. Referring to the grade file keys as the location
# # at which the grades file was found. Generates a blank histogram in cases
# # where no data was found.
# #-----------------------------------------------------------------------------
# for location in found_grade_files.keys():
# for file in range(0, len(found_grade_files[location])):
# # Provide DataFrame to generate empty histograms if no data is found
# if isinstance(found_grade_files[location][file], type(None)):
# grades = pd.DataFrame({'NO DATA FOUND': [-1]})
# else: # Open the grades normally
# grades = pd.read_excel("{}/{}/{}".format(
# self.config.grades_loc,
# location,
# found_grade_files[location][file]
# )
# )
# # Ready the DataFrame for histogramming
# grades = self.ready_DataFrame(
# grades,
# course=row['Course #'],
# term_offered=term_offered
# )
# #-----------------------------------------------------------------------------
# # If the program that is being iterated across does not match the folder that
# # the data was found, that should be indicated. The program entry in the
# # header will always be based on which indicator lookup table the information
# # came from. A note gets added to the header information in the case that data
# # does not come from the same thing. Additionally, the program that the
# # indicator belongs to will always get a copy of the file saved to their
# # histgograms directory.
# #-----------------------------------------------------------------------------
# save_copies_to = [program]
# if location != program:
# indicator_data['Note'] = "Using data from {}".format(location)
# logging.debug("Additional copy of this histogram being saved to %s", location)
# save_copies_to.append(location)
# # Generate the Report object
# logging.debug("Generating histogram from %s/%s/%s",
# self.config.grades_loc,
# location,
# found_grade_files[location][file]
# )
# report = Report.generate_report(indicator_data, bins, self.config, grades)
# #------------------------------------------------------------------------
# # Save the report with a file name reflective of the program, indicator
# # and assessment type to the location that the data came from as well as
# # the program subfolder that the indicator data belongs to. In the case
# # Where multiple grade files were found, add a mumber to the file name.
# #------------------------------------------------------------------------
# # File name formatted string setup
# if len(save_copies_to) > 1:
# # histogram_name = "{pth}/{saveloc}/{prgm} {ind}-{lvl} {crs} {asmt}_{i} {cfg}.pdf"
# # File name that reflects row number in indicator sheet and just the indicator
# histogram_name = "{pth}/{saveloc}/{prgm} {row} {ind}-{lvl} datafrom_{dataloc} {cfg}"
# else:
# # histogram_name= "{pth}/{saveloc}/{prgm} {ind}-{lvl} {crs} {asmt} {cfg}.pdf"
# # File name that reflects row number in indicator sheet and just the indicator
# histogram_name = "{pth}/{saveloc}/{prgm} {row} {ind}-{lvl} {cfg}"
# # Save iteration
# for i in range (0, len(save_copies_to)):
# # File name formatted string formatting
# save_as = histogram_name.format(
# pth = self.config.histograms_loc,
# saveloc = save_copies_to[i],
# prgm = program,
# ind = row["Indicator #"],
# lvl = row["Level"][0],
# crs = row["Course #"],
# asmt = row["Method of Assessment"],
# i = i,
# cfg = self.config.name,
# # row is rownumber + 2 because Pandas indexes from 0 starting from row 2 in
# # the original spreadsheet. zfill adds padding zeros
# row = str(rownumber + 2).zfill(3),
# dataloc = location
# )
# # Make sure the save directory exists, then save the file
# os.makedirs("{pth}/{key}".format(
# pth = self.config.histograms_loc,
# key = save_copies_to[i]
# ),
# exist_ok=True
# )
# report.save(save_as)
logging.info("Autogeneration done!")
def ready_DataFrame(self, grades, course, term_offered):
"""Pretty up a DataFrame for histogramming
This method:
* Turns DataFrame columns into cohort messages to act as legend entries
* Filters cohorts if required
* Indicates if a cohort has no data available
The columns should look something like below normally:
+--------+--------+--------+
| 201603 | 201703 | 201803 |
+========+========+========+
| ... | ... | ... |
+--------+--------+--------+
The method will turn these into cohort messages and use cohort filterings. If
a requested cohort does not appear in the grades files, a No Data column will
get added for that cohort.
Args:
grades(DataFrame): The grades that were just loaded
course(string): The course number (e.g. 'ENGI 3821')
term_offered(int): The term that the course is offered
Returns:
DataFrame: The prettied-up DataFrame
"""
# Try changing the the DataFrame columns to cohort messages.
try:
grades.columns = grades_org.cols_to_cohorts(
grades=grades,
course_name=course,
course_term_offered=term_offered)
# If the columns names were not integer-convertible, append size and move on
except Exception:
logging.debug("Grade columns for this assessment of %s were not integers. " \
+ "Appending column sizes to original column names",
course
)
# Uses a list comprehension to append the column's true size to the message
grades.columns = [
"{crs}({sz})".format(crs=entry,
sz=grades_org.true_size(grades[entry]))
for entry in grades.columns
]
# If a cohort filter is in place, take the columns that have the correct cohort(s)
if self.cohorts is not None:
logging.debug("Detected a cohort filter for cohorts %s",
', '.join([str(c) for c in self.cohorts]))
# Start a list to append relevant DataFrame columns to
new_grades_cols = []
# Iterate across each cohort requested
for c in self.cohorts:
# Iterate across the grades DataFrame columns
occurrences = 0
for col in grades.columns:
if str(c) in col:
new_grades_cols.append(grades[col])
occurrences += 1
logging.debug("Matched cohort %s with column %s",
str(c), col)
# If no matching columns are found, append an NDA column
if occurrences is 0:
logging.warning(
"No grades columns were matched to cohort %s", str(c))
new_grades_cols.append(
pd.Series([-1], name="Co{}: NDA".format(str(c))))
# Create a new DataFrame by concetenating all columns
grades = pd.concat(new_grades_cols,
axis=1,
keys=[s.name for s in new_grades_cols])
return grades
def gen_and_save_histograms(self, dirs_and_files, dir_priorities,
indicator_data, bins, row, rownum, program,
missing_data):
"""Generates and saves histograms for all files passed in
Each file passed to gen_and_save_histograms will have a histogram generated for it
and saved.
Args:
dirs_and_files(dict(list(string))): The grade files that were found. See the
Examples section for some sample inputs
dir_priorities(list(string)): The directory priority to which files should be
saved. In cases where data is not found at the highest priority directory,
but data is found at a lower priority directory, the highest priority
directory will get a copy of the histogram(s) using the data sets in that
low priority directory.
indicator_data(dict(string)): The indicator data for generating Report objects.
The method will pull other required items from here such as Program,
Course Number, etc.
bins(list(int or float)): The bins for generating Report objects
row(pd.Series): The row that the program got all of this indicator data from
rownum(int): The row number that all of this indicator data is coming from
program(string): The program that all of this indicator data belongs to
missing_data: The missing data file
Examples:
dirs_and_files comes in like this (and dir_priorities is in the same order
as the keys)::
{
'ENCM': ['MATH 1001 Course Grade - ENCM.xlsx'],
'Core': [
'MATH 1001 Course Grade - Core.xlsx',
'MATH 1001 Course Grade - Core - By Semester.xlsx'
],
'ECE': ['MATH 1001 Course Grade - ECE.xlsx']
}
Histograms get saved like so:
* ENCM gets a histogram of the one ENCM file
* Core gets 2 histograms of the Core files
* ECE gets 1 histogram of the ECE file
dirs_and_files comes in like this (and dir_priorities is in the same order
as the keys)::
{
'ENCM': [],
'Core': ['MATH 1001 Course Grade - Core.xlsx'],
'ECE': []
}
Histograms get saved like so:
* ENCM gets a copy of a histogram using the data from Core
* Core gets a copy of a histogram using their data, but ENCM's
indicator information
* ECE gets nothing saved to it
dirs_and files comes in like this (and dir_priorities is in the same order
as the keys)::
{
'ENCM': [],
'Core': [
'MATH 1001 Course Grade - Core.xlsx',
'MATH 1001 Course Grade - Core - Program Comparison.xlsx'
],
'ECE': ['MATH 1001 Course Grade - ECE.xlsx']
}
Histograms get saved like so:
* ENCM gets two histograms, both copies of the ones using data from Core
* Core gets two histograms using their data, but with ENCM's indicator
information
* ECE gets one histogram using their data, but with ENCM's indicator
information
dirs_and_files comes in like this (and dir_priorities is in the same order
as the keys)::
{
'ENCM': [],
'Core': [],
'ECE': []
}
Histograms get saved like so:
* Only ENCM gets a histogram. That histogram indicates that no data was
found
"""
logging.info("Preparing to save and generate a list of histograms")
logging.debug("dirs_and_files is %s", str(dirs_and_files))
logging.debug("dir_priorities is %s", str(dir_priorities))
# Keep track of the number of locations that come up empty
empty_locations = 0
# Indicates if an extra copy of the histogram needs to be saved to the top priority dir
save_extra_to = ""
if dirs_and_files[
dir_priorities[0]] == []: # If the top priority dir is empty
logging.debug(
"First priority dir %s is empty. Will save a histogram there",
dir_priorities[0])
save_extra_to = dir_priorities[0]
# Check the unique course table to see if the course has a unique term offering
term_offered = None
for _, unique_course in self.ds.unique_courses.iterrows():
if row['Course #'] == unique_course['Course #']:
term_offered = unique_course['Term Offered']
logging.info("Course %s came up with unique term_offered %s",
row['Course #'], str(term_offered))
break
for dir in dir_priorities:
logging.debug("Generating histograms from dir %s", dir)
if dirs_and_files[dir] == []:
logging.debug("Dir %s is empty. Moving on...", dir)
empty_locations += 1
logging.debug("empty_locations is %i", empty_locations)
else:
# If the dir does not match the program that the indicator belongs to, indicate that
if dir != program:
indicator_data['Notes'] = "Using data from {}".format(dir)
for file in dirs_and_files[
dir]: # For each grades file in the specified dir
# Open the grades file
logging.debug(
"Reading file %s",
"{}/{}/{}".format(self.config.grades_loc, dir, file))
grades = pd.read_excel("{}/{}/{}".format(
self.config.grades_loc, dir, file))
# Ready the DataFrame for histogramming
grades = self.ready_DataFrame(grades,
course=row['Course #'],
term_offered=term_offered)
# Generate a Report object
rprt = Report.generate_report(indicator_data, bins,
self.config, grades)
# File name format if the list of files is only one long
if len(dirs_and_files[dir]) == 1:
histogram_name = "{program} {row} {ind}-{lvl} {cfg}"
# File name format if the list of files is longer
else:
histogram_name = "{program} {row} {ind}-{lvl}_{i} {cfg}"
# Save the Report to the current directory
save_name = histogram_name.format(
program=program,
ind=row["Indicator #"],
lvl=row["Level"][0],
i=dirs_and_files[dir].index(file),
cfg=self.config.name,
# row is rownumber + 2 because Pandas indexes from 0 starting from row 2 in
# the original spreadsheet. zfill adds padding zeros
row=str(rownum + 2).zfill(3))
# Make sure the save directory exists, then save the file
os.makedirs("{path}/{dir}".format(
path=self.config.histograms_loc, dir=dir),
exist_ok=True)
rprt.save("{path}/{dir}/{name}".format(
path=self.config.histograms_loc,
dir=dir,
name=save_name))
# If the extra save location was specified, save a copy there
if save_extra_to:
os.makedirs("{path}/{dir}".format(
path=self.config.histograms_loc, dir=program),
exist_ok=True)
rprt.save("{path}/{dir}/{name}".format(
path=self.config.histograms_loc,
dir=save_extra_to,
name=save_name))
# Set the extra save location back to an empty string to prevent further saving
# Happens outside of the "files in directory" for-loop so that the extra location
# gets copies of every file that matches the criteria. To prevent that behaviour,
# indent by one.
save_extra_to = ""
# If all locations came up empty, generate a blank histogram in the Core directory
if empty_locations == len(dir_priorities):
logging.debug("empty_locations = %i while len(dir_priorities = %i",
empty_locations, len(dir_priorities))
# Write the missing data entry to the missing data file
missing_thing = "{a} {b} ({c}-{d})".format(
a=row["Course #"],
b=row["Method of Assessment"],
c=row['Indicator #'],
d=row['Level'])
logging.warning("No data found for %s", missing_thing)
missing_data.write("Missing data for {}\n".format(missing_thing))
# Set up blank histogram Report
grades = pd.DataFrame({'NO DATA FOUND': [-1]})
indicator_data[
'Notes'] = 'No file containing any of this assessment data was found'
rprt = Report.generate_report(indicator_data, bins, self.config,
grades)
# Set up file name
histogram_name = "{program} {row} {ind}-{lvl} {cfg} NDA"
save_name = histogram_name.format(
program=program,
ind=row["Indicator #"],
lvl=row["Level"][0],
cfg=self.config.name,
# row is rownumber + 2 because Pandas indexes from 0 starting from row 2 in
# the original spreadsheet. zfill adds padding zeros
row=str(rownum + 2).zfill(3))
# Check to see that the directory exists
os.makedirs("{path}/{dir}".format(path=self.config.histograms_loc,
dir=program),
exist_ok=True)
rprt.save("{path}/{dir}/{name}".format(
path=self.config.histograms_loc, dir=program, name=save_name))
# Save histogram to program directory
rprt.save("{path}/{dir}/{name}".format(
path=self.config.histograms_loc, dir=program, name=save_name))
logging.info(
"Finished generating histograms for row %s of %s indicators",
str(rownum), str(program))
@staticmethod
def autogenerate(
config,
programs=None,
cohorts=None,
whitelist=None,
ds=None,
indicators_loc=None,
grades_loc=None,
histograms_loc=None,
):
"""Shortcut to start histogram generation
Initializes a ReportGenerator object and immediately starts report generation.
TODO:
* Create the method
"""
pass
class Downloader(object):
#建立类成员
def __init__(self):
#实例化同级类
self.parser = DataParser()
self.datastore = DataStore()
# 1.通用的网页请求抓取
def get_html(self, url):
try:
# 使用随机User-Agent
ua = UserAgent()
req_headers = {'User-Agent': ua.random}
res = requests.get(url, headers=req_headers)
if res.status_code == requests.codes.ok:
html = res.text
return html
return ''
except Exception as e:
return e
# 2.下载指定作者的所有作品
def downloadworks_oneauthor(self, start_url, authorinfotuple):
# 1）提取作者信息，并设置请求的完整url和结果记录文件名
pagenum = 1
authorid = authorinfotuple[0]
authorname = authorinfotuple[1]
# 2）组成目标页面URL，循环爬行当前作者全部诗文
personalworks_hommeurl = start_url + 'page=%s&id=%s' % (str(pagenum),
authorid)
# 3)遍历所有页面，下载并保存到文件中
try:
# i.爬取个人作品首页，提取总页数
works_html = self.get_html(personalworks_hommeurl)
pagecount = self.parser.getpagecount(works_html)
# ii.创建文档，写入基本信息
totalinfo = u'\n作者:{name},页数：{pagecount}\r\n'.format(
name=authorname, pagecount=pagecount)
path = u'作品集'
filename = path + '\\' + authorname + '.txt'
self.datastore.createfile_oneauther(filename, path, totalinfo)
# iii.遍历作者所有作品页，提取诗文保存到指定文档
for i in range(1, pagecount + 1):
#组合每一页的url
page_url = start_url + 'page=%s&id=%s' % (str(i), authorid)
#请求抓取当前诗文页面
time.sleep(random.randint(3, 6))
singlepageworks_html = self.get_html(page_url)
if len(works_html) > 0:
# 提取当前页中所有诗文
titlelist, contentlist = self.parser.getworks_singlepage(
singlepageworks_html)
# 写入文档
self.datastore.storeworks_singlepage(
filename, i, titlelist, contentlist)
return 'finished'
except Exception as e:
return e
from datetime import datetime
import cPickle as pk
import json
from DataStore import DataStore
# from FHMM import FHMM
# from HMM import HMM, HMM_MAD
from Preprocessing import Appliance, train_test_split, create_matrix
with open('aws_keys.json') as f:
data = json.load(f)
ACCESS_KEY = data['access-key']
SECRET_ACCESS_KEY = data['secret-access-key']
DStore = DataStore('ngalvbucket1', 'house_1')
all_channels = [1, 12, 5, 3, 10, 6, 9, 43, 7, 8]
select_channels = [12, 5, 3, 10, 6, 9, 43, 8]
# select_channels = [12, 5, 6]
DStore.create_store(all_channels)
top_10 = DStore.select_top_k(10,'2013-08-01','2013-09-01')
combined = DStore.create_combined_df('2013-06-01 00:00:00', '2013-10-31 23:59:59', select_channels = select_channels, freq='1Min')
with open('combined.pkl', 'w') as f:
pk.dump(combined,f)
