class VolumeScreen(BaseScreen):
def __init__(self, core):
BaseScreen.__init__(self, core)
self.volume = core.mixer.get_volume().get()
self.progress_bar = ProgressBar(100, DisplayObject.MAX_CHARS_IN_ROW, 5)
self.progress_bar.set_value(self.volume)
def update_display(self, display_object, full_control):
display_object.change_display_data('Volume ' + str(self.volume),
self.progress_bar.get_string())
def on_input_event(self, event):
if event.type == 'click':
if event.key == InputManager.RIGHT:
new_volume = self.volume + 5
if new_volume > 100:
new_volume = 100
self.core.mixer.set_volume(new_volume)
if event.key == InputManager.LEFT:
new_volume = self.volume - 5
if new_volume < 0:
new_volume = 0
self.core.mixer.set_volume(new_volume)
def on_volume_changed(self, volume):
self.volume = volume
self.progress_bar.set_value(self.volume)
self.update = True
def fit(self, inputs, targets, learning_rate, epochs):
# sample dimension first
samples = len(inputs)
# training loop
for i in range(epochs):
err = 0
indicies = np.arange(inputs.shape[0])
np.random.shuffle(indicies)
inputs = inputs[indicies]
targets = targets[indicies]
for j in range(samples):
# forward propagation
output = inputs[j]
for layer in self.layers:
output = layer.forward(output)
# compute loss (for display purpose only)
err += self.Loss(targets[j], output)
# backward propagation
error = self.Loss_der(targets[j], output)
for layer in reversed(self.layers):
error = layer.backpropogate(error, learning_rate)
ProgressBar.printProgressBar(
i*samples+j+1, epochs*samples, 'Training progress', f'Error: {(err/j if j != 0 else 0):.5f}', length=50)
# calculate average error on all samples
err /= samples
def sub():
bar = ProgressBar(max_value=epoch, name=name)
for _ in range(epoch):
time.sleep(cost)
if sub_task is not None:
sub_task()
bar.update()
def createUI(self, oink_widget_list):
self.label = QtGui.QLabel("Choose A Widget:")
self.combo_box_widgets = QtGui.QComboBox()
self.combo_box_widgets.addItems(oink_widget_list)
self.button = QtGui.QPushButton("Launch")
final_layout = QtGui.QHBoxLayout()
final_layout.addWidget(self.label)
final_layout.addWidget(self.combo_box_widgets)
final_layout.addWidget(self.button)
final_page = QtGui.QWidget()
final_page.setLayout(final_layout)
self.progress_bar = ProgressBar()
self.message = QtGui.QLabel("Loading.....")
loading_layout = QtGui.QVBoxLayout()
loading_layout.addWidget(self.progress_bar)
loading_layout.addWidget(self.message)
loading_page = QtGui.QWidget()
loading_page.setLayout(loading_layout)
self.stacked_widget = QtGui.QStackedWidget()
self.stacked_widget.addWidget(final_page)
self.stacked_widget.addWidget(loading_page)
layout = QtGui.QVBoxLayout()
layout.addWidget(self.stacked_widget)
layout.addWidget(Taunter())
self.setLayout(layout)
self.setWindowTitle("OINK Widget Chooser")
icon_file_name_path = os.path.join(MOSES.getPathToImages(),
'PORK_Icon.png')
self.setWindowIcon(QtGui.QIcon(icon_file_name_path))
self.show()
def ProduceSiteReadinessSSBFiles(self):
print "\nProducing Site Readiness SSB files to commission view\n"
prog = ProgressBar(0, 100, 77)
for dayspan in 30, 15, 7:
prog.increment(100./3.)
fileSSBRanking = self.ssbOutDir + '/SiteReadinessRanking_SSBfeed_last' + str(dayspan) + 'days.txt'
fileHandle = open ( fileSSBRanking , 'w' )
sitesit = self.matrices.readiValues.keys()
sitesit.sort()
for sitename in sitesit:
if self.SkipSiteOutput(sitename): continue
pl = "R+Wcorr_perc"
color = "red"
if sitename.find("T1") == 0 and self.matrices.stats[sitename][dayspan][pl]>90:
color="green"
if sitename.find("T2") == 0 and self.matrices.stats[sitename][dayspan][pl]>80:
color="green"
if self.matrices.stats[sitename][dayspan][pl] != "n/a":
filenameSSB = self.options.url + "/SiteReadiness/PLOTS/" + sitename.split("_")[0] + "_" + pl + "_last" + str(dayspan) + "days_" + self.tinfo.timestamphtml + ".png"
tofile = self.tinfo.todaystampfileSSB + '\t' + sitename + '\t' + str(self.matrices.stats[sitename][dayspan][pl]) + '\t' + color + '\t' + filenameSSB + "\n"
fileHandle.write(tofile)
fileHandle.close()
prog.finish()
def parse(self):
'''
return parsed data.
{
'heading': headingStr,
'nodes': [Node],
'elementGroups': [ElementGroups],
'loads': [Load]
}
'''
with open(self.fin, 'r') as fp:
self.lines = fp.readlines()
self.bar = ProgressBar(1000, printCount=False, printTime=True)
self.index = -1
self.parseHeading()
self.parseParts()
self.parseAssembly()
self.parseMaterials()
self.parseLoad()
self.bar.update(self.bar.maxCount)
del self.bar
del self.lines
self.analyse()
return self.data()
def ProduceSiteReadinessSSBFile(self):
print "\nProducing Site Readiness SSB input file\n"
prog = ProgressBar(0, 100, 77)
fileHandle = open(self.fileSSB, "w")
sitesit = self.matrices.readiValues.keys()
sitesit.sort()
for sitename in sitesit:
prog.increment(100.0 / len(sitesit))
if self.SkipSiteOutput(sitename):
continue
status = self.matrices.readiValues[sitename][self.tinfo.yesterdaystamp]
colorst = self.SRMatrixColors[status]
linkSSB = self.options.url + "/SiteReadiness/HTML/SiteReadinessReport_" + self.tinfo.timestamphtml + ".html"
tofile = (
self.tinfo.todaystampfileSSB
+ "\t"
+ sitename
+ "\t"
+ status
+ "\t"
+ colorst
+ "\t"
+ linkSSB
+ "#"
+ sitename
+ "\n"
)
fileHandle.write(tofile)
fileHandle.close()
prog.finish()
def PrintDailyMetrics(self):
prog = ProgressBar(0, 100, 77)
indmetrics = self.cinfo.metorder.keys()
indmetrics.sort()
sites = self.matrices.columnValues.keys()
sites.sort()
for sitename in sites:
prog.increment(100./3.)
dates = self.matrices.dailyMetrics[sitename].keys()
dates.sort()
for dat in dates:
if self.SkipSiteOutput(sitename): continue
for metnumber in indmetrics:
met = self.cinfo.metorder[metnumber] #colName
met1 = self.cinfo.printCol[met] #pCol (print permission)
#if not self.matrices.columnValues[sitename][dat].has_key(met) or met == 'IsSiteInSiteDB': continue # ignore
if not self.matrices.columnValues[sitename][dat].has_key(met) or met1 == '0' : continue # ignore
if self.matrices.columnValues[sitename][dat][met].has_key('URL'):
url = self.matrices.columnValues[sitename][dat][met]['URL']
else:
url = "-"
print dat, sitename, met, self.matrices.columnValues[sitename][dat][met]['Status'], self.matrices.columnValues[sitename][dat][met]['Color'],url
prog.finish()
def processFiles(self, files, folderPath):
"""function processFiles
returns []
"""
if _logger.getEffectiveLevel() == 30:
import os
sumSize = 0
for file in files:
sumSize += os.path.getsize(folderPath+file)
bar = ProgressBar(sumSize)
else:
bar = False
#Creating MULTI FASTA file, with all genens
import os
fastaFile = open(os.path.expanduser("~/Documents/th6_MaartenJoost_multi.fasta"), 'w')
# evt [http://docs.python.org/2/library/multiprocessing.html]
for file in sorted(files):
self.chromosomeNumber += 1
chromosome = Chromosome(file, folderPath, _DB, _logger, fastaFile)
self.oligosNumber += chromosome.oligosNumber
self.genesNumber += chromosome.genesNum
del(chromosome)
if bar: bar.update(os.path.getsize(folderPath+file))
if bar: del(bar)
def kmeans(self):
km_name = []
km_data = []
case_dict = self.__storage.get_case_dict()
progressbar = ProgressBar('计算题目难度', 1)
for case_id in case_dict.keys():
km_name.append(case_id)
records = np.array(case_dict[case_id]['records'])
scores = np.array(case_dict[case_id]['scores'])
km_data.append([
records.mean(), case_dict[case_id]['avg_times'],
records.std(),
scores.std(),
scores.mean()
])
km_data = np.array(km_data)
km = KMeans(n_clusters=5)
label = km.fit_predict(km_data)
centers = km.cluster_centers_
case_cluster = [{
'center': centers[i],
'cases': []
} for i in range(0, 5)]
for i in range(0, len(km_name)):
case_cluster[label[i]]['cases'].append(km_name[i])
case_cluster = self.__cluster_sort(case_cluster)
progressbar.progress()
self.__storage.set_case_cluster(case_cluster)
self.__storage.set_km_data(km_data)
self.__storage.set_label(label)
def grouping(self):
# 用户、题目分为5个组
case_group = [[], [], [], [], []]
user_group = [[], [], [], [], []]
# 字典 —— 根据题目号、用户号找组号
case_group_dict = {}
user_group_dict = {}
data = self.__storage.get_data()
progressbar = ProgressBar('分组中', 5)
for user_data in data:
user_id = user_data['user_id']
# 5个用户代表5个组，手动抽取的，根据这5个用户找出每个组分的题是什么，谁分到了哪个组
if user_id in (60636, 60762, 60692, 49823, 58616):
for case in user_data['cases']:
i = len(user_group_dict)
case_id = case['case_id']
case_type = case['case_type']
case_id = case_id + case_type
if case_id in case_group_dict:
case_group_dict[case_id].append(i)
else:
case_group_dict[case_id] = [i]
case_group[i].append(case_id)
user_group_dict[user_id] = len(user_group_dict)
progressbar.progress()
self.__storage.set_case_group(case_group)
self.__storage.set_user_group(user_group)
self.__storage.set_case_group_dict(case_group_dict)
self.__storage.set_user_group_dict(user_group_dict)
def Test(self, length, max_days_ago=600, num_agents=1):
scores = []
current = get_current()
for _ in range(len(self.agents)):
scores.append(0)
for i in range(length):
daysAgo = np.random.randint(0, max_days_ago)
for agent in range(len(self.agents)):
with HiddenPrints():
self.agents[agent].Test(
1,
dt.utcnow() - timedelta(days=daysAgo), False)
scores[agent] += self.agents[agent].get_score(current)
ProgressBar.printProgressBar(i * len(self.agents) + agent + 1,
length * len(self.agents),
prefix='Testing',
length=50)
if num_agents == 1:
best_agent = self.agents[scores.index(max(scores))]
return best_agent
else:
best_agents = sorted(
self.agents,
key=lambda agent: scores[self.agents.index(agent)],
reverse=True)
return best_agents[:num_agents]
def play_games(self):
"""
Plays the amount of games that is given by game_iterations.
If self.verbose is set to False, the progress of the games played will be shown.
Lastly, the result statistics of the games played will be printed.
"""
player1_wins = 0
player1_starts = 0
if not self.verbose:
progress = ProgressBar(self.game_iterations, "Playing games:")
for i in range(self.game_iterations):
self.tree.clean() # <- No inter game learning
self.tree.reset_exploration()
self.simulate_games()
game = self.create_game(SIM=False)
if game.current_player == 1:
player1_starts += 1
if self.verbose:
print(f"\nGame {i+1} of {self.game_iterations}:\n")
self.tree.exploit()
self.play_game(game)
if self.verbose:
print()
else:
progress.show(i)
if game.get_winner() == 1:
player1_wins += 1
print(f'\nPlayer 1 started {player1_starts} of {self.game_iterations} games ({player1_starts/self.game_iterations*100:.1f}%).')
print(f'Player 1 won {player1_wins} of {self.game_iterations} games ({player1_wins/self.game_iterations*100:.1f}%).\n')
def main(iterations, ensembles, episodes, agent_type, agent_args, env_type,
env_args, file_prefix):
"""RL Testbed.
@arg iterations: Number of environments to average over
@arg ensembles: Number of bots to average over
@arg episodes: Number of episodes to run for
@arg agent_type: String name of agent
@arg agent_args: Arguments to the agent constructor
@arg env_type: String name of environment
@arg env_args: Arguments to the environment constructor
"""
# Load agent and environment
progress = ProgressBar(0, ensembles * iterations, mode='fixed')
# Needed to prevent glitches
oldprog = str(progress)
# Counters
ret = np.zeros(episodes, dtype=float)
min_, max_ = np.inf * \
np.ones(episodes, dtype=float), -np.inf * \
np.ones(episodes, dtype=float)
var = np.zeros(episodes, dtype=float)
env = env_type.create(*env_args)
for i in xrange(1, iterations + 1):
env = env.domain.reset_rewards(env, *env_args)
ret_ = np.zeros(episodes, dtype=float)
# Initialise environment and agent
for j in xrange(1, ensembles + 1):
agent = agent_type(env.Q, *agent_args)
ret__ = Runner.run(env, agent, episodes)
ret__ = np.cumsum(ret__)
# Add to ret_
ret_ += (ret__ - ret_) / j
# print progress
progress.increment_amount()
if oldprog != str(progress):
print progress, "\r",
sys.stdout.flush()
oldprog = str(progress)
ret += (ret_ - ret) / i
min_ = np.min(np.vstack((min_, ret_)), axis=0)
max_ = np.max(np.vstack((max_, ret_)), axis=0)
var_ = np.power(ret_, 2)
var += (var_ - var) / i
print "\n"
var = np.sqrt(var - np.power(ret, 2))
f = open("%s-return.dat" % (file_prefix), "w")
# Print ret
for i in xrange(len(ret)):
f.write("%d %f %f %f %f\n" % (i + 1, ret[i], min_[i], max_[i], var[i]))
f.close()
def PrintDailyMetricsStats(self):
print "\nPrinting Daily Metrics Statistics\n"
prog = ProgressBar(0, 100, 77)
fileHandle = open(self.asciiOutDir + "/Daily_HistoricStatistics.txt", "w")
sites = self.matrices.dailyMetrics.keys()
sites.sort()
for sitename in sites:
dates = self.matrices.dailyMetrics[sitename].keys()
dates.sort()
continue
for i in "T1", "T2":
prog.increment(100.0 / 2.0)
for dat in dates:
countO = 0
countE = 0
countSD = 0
countna = 0
for sitename in sites:
if sitename.find("T1_CH_CERN") == 0:
continue
if not sitename.find(i + "_") == 0:
continue
if self.SkipSiteOutput(sitename):
continue
state = self.matrices.dailyMetrics[sitename][dat]
if state == "O":
countO += 1
if state == "E":
countE += 1
if state.find("n/a") == 0:
countna += 1
if state == "SD":
countSD += 1
if dat == self.tinfo.todaystamp:
continue
tofile = (
"Daily Metric "
+ i
+ " "
+ dat
+ " "
+ str(countE)
+ " "
+ str(countO)
+ " "
+ str(countna)
+ " "
+ str(countSD)
+ " "
+ str(countE + countO + countSD + countna)
+ "\n"
)
fileHandle.write(tofile)
fileHandle.close()
prog.finish()
def ParseXML(self):
print "\nObtaining XML info from SSB 'Site Readiness' view\n"
prog = ProgressBar(0, 100, 77)
xmlCacheDir = self.options.path_out + "/INPUTxmls"
if not os.path.exists(xmlCacheDir):
os.makedirs(xmlCacheDir)
ColumnItems = self.cinfo.urls.keys()
ColumnItems.sort()
for col in ColumnItems:
prog.increment(100./len(ColumnItems))
url = self.cinfo.urls[col]
xmlFile = xmlCacheDir + "/" + col + ".xml"
if self.options.xml == 'false' and not os.path.exists(xmlCacheDir):
print "\nWARNING: you cannot re-use the XML files as the files were not obtained before. Obtaining them...\n"
self.options.xml = 'true'
if self.options.xml == 'true': # download xml file if requested
print "Column %s - Getting the url %s" % (col, url)
os.system("curl -s -H 'Accept: text/xml' '%s' > %s" % (url,xmlFile))
f = file(xmlFile,'r') # read xml file that was either just written, or was written in the previous run
t = xml.dom.minidom.parse(f)
f.close()
#print t.toprettyxml()
for subUrl in xpath.Evaluate("/getplotdata/csvdata/item", t):
#print subUrl.toprettyxml()
info = {} # basic info about the site for this column
for option in ('Status', "COLOR", 'Time', 'EndTime','VOName','URL'):
for target in xpath.Evaluate(option, subUrl):
if target.hasChildNodes():
s = target.firstChild.nodeValue.encode('ascii')
else:
s = ""
info[option] = s
voname = info['VOName']
time = info['Time']
xmlMatrix = self.matrices.xmlInfo
if self.options.oneSite != "" and voname.find(self.options.oneSite) != 0: continue
if not xmlMatrix.has_key(voname): # if site not already in dict, add an empty dict for it
xmlMatrix[voname] = {}
if not xmlMatrix[voname].has_key(col): # if site entry doesn't already have this column, add an empty dict for this column
xmlMatrix[voname][col] = {}
xmlMatrix[voname][col][time] = info # set the actual values
# Correct some of the strings
value = xmlMatrix[voname][col][time]['Status']
if col=="HammerCloud" and value != "n/a":
value = str(int(float(value)))
if value.find("%") != 0:
value += "%"
elif col=="SUMAvailability":
value = str(int(round(float(value))))
if value.find("%") != 0:
value += "%"
xmlMatrix[voname][col][time]['Status'] = value
prog.finish()
def main( iterations, ensembles, episodes, agent_type, agent_args, env_type, env_args, file_prefix ):
"""RL Testbed.
@arg iterations: Number of environments to average over
@arg ensembles: Number of bots to average over
@arg episodes: Number of episodes to run for
@arg agent_type: String name of agent
@arg agent_args: Arguments to the agent constructor
@arg env_type: String name of environment
@arg env_args: Arguments to the environment constructor
"""
# Load agent and environment
progress = ProgressBar( 0, ensembles*iterations, mode='fixed' )
# Needed to prevent glitches
oldprog = str(progress)
# Counters
ret = np.zeros( episodes, dtype=float )
min_, max_ = np.inf * np.ones( episodes, dtype=float) , -np.inf * np.ones( episodes, dtype=float)
var = np.zeros( episodes, dtype=float )
env = env_type.create( *env_args )
for i in xrange( 1, iterations+1 ):
env = env.domain.reset_rewards( env, *env_args )
ret_ = np.zeros( episodes, dtype=float )
# Initialise environment and agent
for j in xrange( 1, ensembles+1 ):
print "iter, ens", i, j
agent = agent_type( env.Q, *agent_args )
ret__ = Runner.run( env, agent, episodes )
ret__ = np.cumsum( ret__ ) #Varun
# Add to ret_
ret_ += (ret__ - ret_) / j
# print progress
progress.increment_amount()
if oldprog != str(progress):
print progress, "\r",
sys.stdout.flush()
oldprog=str(progress)
ret += (ret_ - ret) / i
min_ = np.min( np.vstack( ( min_, ret_ ) ), axis=0 )
max_ = np.max( np.vstack( ( max_, ret_ ) ), axis=0 )
var_ = np.power( ret_, 2 )
var += (var_ - var) / i
print "\n"
var = np.sqrt( var - np.power( ret, 2 ) )
f = open("%s-return.dat"%( file_prefix ), "w")
# Print ret
for i in xrange( len( ret ) ):
f.write( "%d %f %f %f %f\n"%( i+1, ret[ i ], min_[i], max_[i], var[ i ] ) )
f.close()
def progressButtonClicked(self, music):
self.calc = ProgressBar()
self.calc.countChanged.connect(self.changeThePbarValue)
self.calc.musicLengthSignal.emit(music.__len__(music.getFile()))
self.pBar.setMaximum(music.__len__(music.getFile()))
self.durationMusicLabel.setText(
self.calculateTheDuration(music, music.getFile()))
self.calc.moveToTheNextMusic.connect(self.moveNextMusic)
self.calc.start()
class RawDataManager(QtGui.QWidget):
def __init__(self, user_id, password, *args, **kwargs):
super(RawDataManager, self).__init__(*args, **kwargs)
self.user_id, self.password = user_id, password
self.raw_data_thread = RawDataUploaderThread(self.user_id, self.password)
self.createUI()
self.mapEvents()
def createUI(self):
#self.raw_data_filter_form = FilterForm()
self.fsn_entry_field = FSNTextEdit()
self.raw_data_table = CopiableQTableWidget(0,0)
self.upload_raw_data_button = QtGui.QPushButton("Upload Raw Data from File")
self.progress_bar = ProgressBar()
self.progress_log = QtGui.QTextEdit()
layout = QtGui.QVBoxLayout()
layout.addWidget(self.upload_raw_data_button)
layout.addWidget(self.progress_bar)
layout.addWidget(self.progress_log)
self.setLayout(layout)
self.setWindowTitle("Raw Data Uploader")
self.setWindowIcon(QtGui.QIcon(os.path.join(MOSES.getPathToImages(),"PORK_Icon.png")))
self.show()
def mapEvents(self):
self.upload_raw_data_button.clicked.connect(self.uploadRawData)
self.raw_data_thread.sendActivity.connect(self.displayActivity)
self.raw_data_thread.sendMessage.connect(self.displayMessage)
def displayActivity(self, progress, eta, accepted, rejected, failed, pending):
self.progress_bar.setValue(progress)
message = "%d Accepted, %d Rejected, %d Failed, %d Pending. ETA: %s"%(accepted, rejected, failed, pending, eta)
self.displayMessage(message)
if progress == 100:
self.upload_raw_data_button.setEnabled(True)
self.alertMessage("Completed uploading raw data!", "%s"%message)
def displayMessage(self, message):
self.progress_log.append("%s: <b>%s</b>"%(datetime.datetime.now(),message))
self.progress_log.moveCursor(QtGui.QTextCursor.End)
def uploadRawData(self):
self.upload_raw_data_button.setEnabled(False)
data_file_name = str(QtGui.QFileDialog.getOpenFileName(self,"Open Data File",os.getcwd(),("MS Excel Spreadsheet (*.xlsx)")))
if data_file_name is not None:
if os.path.exists(data_file_name):
xl_file = pd.ExcelFile(data_file_name)
if "Raw Data" in xl_file.sheet_names:
raw_data = xl_file.parse("Raw Data")
self.raw_data_thread.setDataFrame(raw_data)
else:
self.alertMessage("Invalid raw data file.","""The given raw data file doesn't seem to have any sheet named "Raw Data".""")
def alertMessage(self, title, message):
QtGui.QMessageBox.about(self, title, message)
def calculateForce(self):
load = Load()
self.bar = ProgressBar(1000, printCount=False, printTime=True)
fs = dict() # fs marks every direction's force as {1:2.0, 2:3.0}
for i in range(3):
if self.dload.direction[i]:
fs[i + 1] = self.dload.mag * self.dload.direction[i]
# {index: {direction: force, }}
self.globalForcesByGlobalNodeIndexDict = dict()
self.bar.count = 0
for insIndex in self.instancesDict:
ins = self.instancesDict[insIndex]
part = ins.part
def run(elementIndex):
element = part.localElementsDict[elementIndex]
# calculate a unit body force at each node
# returns as {1:3.2, 2:2.4}
nodeForces = self.calculateBodyForceAtElement(element, ins)
for localNodeIndex in nodeForces:
globalNode = ins.globalNodesDict[localNodeIndex]
for direction in fs:
# get forces for global node
forces = self.globalForcesByGlobalNodeIndexDict.get(
globalNode.index)
if not forces:
forces = dict()
self.globalForcesByGlobalNodeIndexDict[
globalNode.index] = forces
# get direction force from this node forces
force = forces.get(direction)
if not force:
force = Force()
force.direction = direction
force.node = globalNode
forces[direction] = force
load.forces.append(force) # append new force only
force.mag += fs[direction] * nodeForces[localNodeIndex]
self.bar.count += 1
if self.bar.count / self.elementCount > self.bar.currentCount / self.bar.maxCount:
self.bar.grow()
# for elementIndex in part.localElementsDict:
pool = Pool()
pool.map(run, part.localElementsDict)
pool.close()
self.loads.append(load)
del self.bar
def ProduceSiteReadinessStatistics(self):
print "\nProducing Site Readiness Statistics\n"
sitesit = self.matrices.readiValues.keys()
sitesit.sort()
prog = ProgressBar(0, 100, 77)
for dayspan in 30, 15, 7:
prog.increment(100./3.)
for sitename in sitesit:
if self.SkipSiteOutput(sitename): continue
countR = 0; countW = 0; countNR = 0; countSD = 0; countNA = 0
infostats2 = {}
if not self.matrices.stats.has_key(sitename):
self.matrices.stats[sitename]={}
for i in range(0,dayspan):
deltaT = datetime.timedelta(i)
datestamp = self.tinfo.yesterday - deltaT
state = self.matrices.readiValues[sitename][datestamp.strftime("%Y-%m-%d")]
if state == "R": countR += 1
if state == "W": countW += 1
if state == "NR": countNR += 1
if state == "SD": countSD += 1
if state.find("n/a") == 0: countNA += 1
if not self.matrices.stats[sitename].has_key(dayspan):
self.matrices.stats[sitename][dayspan]={}
infostats2['R_perc']= (int)(round(100.*countR/dayspan))
infostats2['W_perc']= (int)(round(100.*countW/dayspan))
infostats2['R+W_perc']= (int)(round(100.*(countR+countW)/dayspan))
infostats2['NR_perc']= (int)(round(100.*countNR/dayspan))
infostats2['SD_perc']= (int)(round(100.*countSD/dayspan))
infostats2['R']= countR
infostats2['W']= countW
infostats2['R+W']= countW+countR
infostats2['NR']= countNR
infostats2['SD']= countSD
infostats2['days']=dayspan
if (dayspan-countSD-countNA)!=0:
infostats2['Rcorr_perc']= (int)(round(100.*countR/(dayspan-countSD-countNA)))
infostats2['Wcorr_perc']= (int)(round(100.*countW/(dayspan-countSD-countNA)))
infostats2['R+Wcorr_perc']= (int)(round(100.*(countR+countW)/(dayspan-countSD-countNA)))
infostats2['NRcorr_perc']= (int)(round(100.*countNR/(dayspan-countSD-countNA)))
else:
infostats2['Rcorr_perc']= 0
infostats2['Wcorr_perc']= 0
infostats2['R+Wcorr_perc']= 0
infostats2['NRcorr_perc']= 100
self.matrices.stats[sitename][dayspan]=infostats2
prog.finish()
def reset(self, title, indeterminate=False):
if indeterminate != self.indeterminate:
if self.bar: self.bar.finish()
self.indeterminate = indeterminate
if self.bar:
self.bar = ProgressBar(title=title,
indeterminate=indeterminate)
self.progBar = "[]" # This holds the progress bar string
self.width = 40
self.amount = 0 # When amount == max, we are 100% done
class y_sampler(object):
def __init__(self, path_planner, path):
parameters = path_planner.parameters
self.X = path_planner.X
self.E = parameters.E
self.N = parameters.N
self.hazards = parameters.hazards
self.sample_Tau_Ys = parameters.sample_Tau_Ys
print("...Reading samples...")
if parameters.samples_file["Read"]:
infile = open(path + parameters.samples_file["Name"], 'rb')
self.episodes = pickle.load(infile)
infile.close()
else:
self.progress_bar = ProgressBar(self.N)
print("...Monte Carlo sampling...")
self.episodes = self.generate_episodes()
self.add_cheat_sample()
print("...Saving samples...")
outfile = open(path + parameters.samples_file["Name"], 'wb')
pickle.dump(self.episodes, outfile)
outfile.close()
def generate_episodes(self):
episodes = np.zeros((self.E, self.N, len(self.X)), dtype=bool)
for hazard in self.hazards:
y_0_h = hazard.y_0
p_f_h = hazard.p_f
episodes_h = self.init_episodes(y_0_h)
for t in range(1, self.N):
ys_t_1 = episodes_h[:, t - 1, :]
ys_t = self.sample_Tau_Ys(self, p_f_h, ys_t_1)
episodes_h[:, t, :] = ys_t
if t % int(self.N / self.N) == 0:
self.progress_bar.progress(t)
self.progress_bar.progress(self.N, 'Finished!\n')
episodes = episodes + episodes_h
return episodes
def init_episodes(self, y_0):
episodes = np.zeros((self.E, self.N, len(self.X)), dtype=bool)
for e in y_0:
episodes[:, 0, self.X.index(e)] = True
return episodes
def add_cheat_sample(self):
y_0 = self.episodes[0, 0, :]
self.episodes[0, :, :] = np.kron(np.ones((self.N, 1), dtype=bool), y_0)
def perfromSubmission(self,matched,task):
njs=0
### Progress Bar indicator, deactivate for debug
if common.debugLevel == 0 :
term = TerminalController()
if len(matched)>0:
common.logger.info(str(len(matched))+" blocks of jobs will be submitted")
common.logger.debug("Delegating proxy ")
try:
common.scheduler.delegateProxy()
except CrabException:
common.logger.debug("Proxy delegation failed ")
for ii in matched:
common.logger.debug('Submitting jobs '+str(self.sub_jobs[ii]))
# fix arguments for unique naming of the output
common._db.updateResubAttribs(self.sub_jobs[ii])
try:
common.scheduler.submit(self.sub_jobs[ii],task)
except CrabException:
common.logger.debug('common.scheduler.submit exception. Job(s) possibly not submitted')
raise CrabException("Job not submitted")
if common.debugLevel == 0 :
try: pbar = ProgressBar(term, 'Submitting '+str(len(self.sub_jobs[ii]))+' jobs')
except: pbar = None
if common.debugLevel == 0:
if pbar :
pbar.update(float(ii+1)/float(len(self.sub_jobs)),'please wait')
### check the if the submission succeded Maybe not needed or at least simplified
sched_Id = common._db.queryRunJob('schedulerId', self.sub_jobs[ii])
listId=[]
run_jobToSave = {'status' :'S'}
listRunField = []
for j in range(len(self.sub_jobs[ii])):
if str(sched_Id[j]) != '':
listId.append(self.sub_jobs[ii][j])
listRunField.append(run_jobToSave)
common.logger.debug("Submitted job # "+ str(self.sub_jobs[ii][j]))
njs += 1
common._db.updateRunJob_(listId, listRunField)
self.stateChange(listId,"SubSuccess")
self.SendMLpost(self.sub_jobs[ii])
else:
common.logger.info("The whole task doesn't found compatible site ")
return njs
def performSubmission(self,matched,task):
njs=0
### Progress Bar indicator, deactivate for debug
if common.debugLevel == 0 :
term = TerminalController()
if len(matched)>0:
common.logger.info(str(len(matched))+" blocks of jobs will be submitted")
common.logger.debug("Delegating proxy ")
try:
common.scheduler.delegateProxy()
except CrabException:
common.logger.debug("Proxy delegation failed ")
for ii in matched:
common.logger.debug('Submitting jobs '+str(self.sub_jobs[ii]))
# fix arguments for unique naming of the output
common._db.updateResubAttribs(self.sub_jobs[ii])
try:
common.scheduler.submit(self.sub_jobs[ii],task)
except CrabException:
common.logger.debug('common.scheduler.submit exception. Job(s) possibly not submitted')
raise CrabException("Job not submitted")
if common.debugLevel == 0 :
try: pbar = ProgressBar(term, 'Submitting '+str(len(self.sub_jobs[ii]))+' jobs')
except: pbar = None
if common.debugLevel == 0:
if pbar :
pbar.update(float(ii+1)/float(len(self.sub_jobs)),'please wait')
### check the if the submission succeded Maybe not needed or at least simplified
sched_Id = common._db.queryRunJob('schedulerId', self.sub_jobs[ii])
listId=[]
run_jobToSave = {'status' :'S'}
listRunField = []
for j in range(len(self.sub_jobs[ii])):
if str(sched_Id[j]) != '':
listId.append(self.sub_jobs[ii][j])
listRunField.append(run_jobToSave)
common.logger.debug("Submitted job # "+ str(self.sub_jobs[ii][j]))
njs += 1
common._db.updateRunJob_(listId, listRunField)
self.stateChange(listId,"SubSuccess")
self.SendMLpost(self.sub_jobs[ii])
else:
common.logger.info("The whole task doesn't found compatible site ")
return njs
def __init__(self):
self.data = self.load_data('data/movies_metadata.csv').fillna('')
self.stop_words = stopwords.words('english')
# unique tokens in the dataset
self.unique_tokens = set()
# dict of tokens appearing in number of documents
self.document_frequency = {}
# dict of tokens appearing in document with its frequency
self.postings = {}
# dict of number of tokens in each document (not unique)
self.lengths = {}
self.progress = ProgressBar()
def showProgress(self, now, all):
if self.ProgressWindow == None:
self.ProgressWindow = ProgressBar()
self.ProgressWindow.show()
qe = QEventLoop()
qe.exec_()
else:
self.ProgressWindow.show()
self.ProgressWindow.getPercent(now, all)
if now == all:
self.ProgressWindow.hide()
app._notifications_.showNotification("极客宝宝", "osc_tweet_notice",
"主人，您的数据已经下载完成哦！！！", 30000)
def indexNodes(self):
self.linkedNodes = []
self.nodeCount = 0
self.localPointsSum = sum(
len(self.instancesDict[insindex].part.localNodesDict)
for insindex in self.instancesDict)
localPointsCount = 0
self.bar = ProgressBar(1000, printCount=False, printTime=True)
self.bar.begin()
for insName in self.instancesDict:
instance = self.instancesDict[insName]
part = instance.part
for nodeLocalIndex in part.localNodesDict:
localNode = part.localNodesDict[nodeLocalIndex]
globalPos = calculatePos(instance, localNode)
if isLocalNodeIndexInInstanceNsets(nodeLocalIndex, instance):
globalNode = self.matchGlobalNode(globalPos)
if globalNode: # already counted
instance.globalNodesDict[nodeLocalIndex] = globalNode
else: # new boundary node
self.nodeCount += 1
globalNode = Node(globalPos)
globalNode.index = self.nodeCount
globalNode.bounds = (0, 0, 0)
instance.globalNodesDict[nodeLocalIndex] = globalNode
self.globalNodesDict[self.nodeCount] = globalNode
self.linkedNodes.append(globalNode)
else: # inside nodes
self.nodeCount += 1
globalNode = Node(globalPos)
globalNode.index = self.nodeCount
globalNode.bounds = (0, 0, 0)
instance.globalNodesDict[nodeLocalIndex] = globalNode
self.globalNodesDict[self.nodeCount] = globalNode
localPointsCount += 1
if localPointsCount / self.localPointsSum > \
self.bar.currentCount / self.bar.maxCount:
self.bar.grow()
for bound in self.boundaries:
for nset in bound.nsets:
for nodeIndex in nset.nodeIndexs:
node = nset.instance.globalNodesDict[nodeIndex]
b = list(node.bounds)
b[bound.displacementIndex - 1] = 1
node.bounds = tuple(b)
del self.bar
def calc_user_result(self):
types = ['字符串', '线性表', '数组', '查找算法', '排序算法', '数字操作', '树结构', '图结构']
weight = [2, 1.5, 1, 0.8, 0.5]
group_weight = self.__storage.get_group_weight()
user_dict = self.__storage.get_user_dict()
case_dict = self.__storage.get_case_dict()
user_group_dict = self.__storage.get_user_group_dict()
case_cluster = self.__storage.get_case_cluster()
user_result = {}
progressbar = ProgressBar('计算用户得分结果', len(user_dict))
for user_id in user_dict.keys():
user = {}
cases = user_dict[user_id]
for case_id in cases.keys():
case_type = cases[case_id]['type']
final_score = cases[case_id]['final_score']
if not case_type in user:
user[case_type] = []
case_weight = -1
for i in range(0, 5):
if case_id in case_cluster[i]['cases']:
case_weight = weight[i]
break
a = np.array(case_dict[case_id]['records'])
# 用户在这题的提交平均分和所有人在这题的提交平均分之差
score_d = (cases[case_id]['avg_score'] - a.mean()) * 0.5
# 所有人在这题的提交平均提交次数和用户在这题的提交次数之差
times_d = case_dict[case_id]['avg_times'] - cases[case_id][
'record_num']
# 以本题final_score为主要依据，根据上述两个差对得分进行浮动
score = final_score + (min(15, score_d)
if score_d >= 0 else max(-15, score_d))
score = score + (min(5, times_d) if times_d >= 0 else max(
-5, times_d))
score = score + cases[case_id]['lint_score']
# 得分区间在[0,100]内，再乘以权重
score = min(100, max(0, score / 1.3)) * case_weight
user[case_type].append(score)
user_result[user_id] = {}
group_id = user_group_dict[user_id]
for case_type in user.keys():
total_score = 0
total_weight = group_weight[group_id][case_type]
for score in user[case_type]:
total_score += score
user_result[user_id][case_type] = total_score / total_weight
for ty in types:
if ty not in user_result[user_id]:
user_result[user_id][ty] = 0
progressbar.progress()
self.__storage.set_user_result(user_result)
def __init__(self, core):
BaseScreen.__init__(self, core)
self.progress_bar = ProgressBar(0, DisplayObject.MAX_CHARS_IN_ROW, 5)
self.reference_second = 0
self.second_in_song = 0
self.current_second = 0
self.track_current_second_text = None
self.track_length_text = None
self.length = None
self.paused = False
def train(self):
loss_statistics = []
acc_statistics = []
# TRAINING EPOCHS
for epoch in range(self.EPOCHS):
# Diagnostic Data
print('EPOCH', epoch + 1, '/', self.EPOCHS)
# Progress Bar object
pb = ProgressBar()
# Statistics
loss_data = []
acc_data = []
# For each training example
for i, batch in enumerate(self.dataloader):
# Create message and label tensors
msg_tensor, lbl_tensor = self.prep_tensors(batch)
# Send data through neural net, calculate loss, and back propagate
self.optimizer.zero_grad()
output = self.network(msg_tensor)
loss = self.criterion(output, lbl_tensor.type_as(output))
loss.backward()
self.optimizer.step()
# Calculate Statistics
loss_data.append(float(loss))
acc_data.append(self.calc_acc(output, lbl_tensor))
# Print batch diagnostics
if i % self.BATCH_DIAGNOSTICS == 0:
prefix = f'Epoch Progress'
suffix = f'[Batch {i}/{self.DATASET_SIZE // self.BATCH_SIZE}] ({self.BATCH_SIZE}) ' \
f'| Loss : {round(mean(loss_data), 5)} ' \
f'| Acc : {round(mean(acc_data) * 100, 2)}%'
pb.printProgressBar(i + 1,
self.DATASET_SIZE // self.BATCH_SIZE,
prefix=prefix,
suffix=suffix,
length=30)
loss_statistics.append(loss_data)
acc_statistics.append(acc_data)
print()
self.display_statistics(loss_statistics, acc_statistics)
def __init__(self):
gtk.Frame.__init__(self)
self.set_label(" Acciones: ")
self.set_border_width(5)
self.modify_bg(0, get_colors("toolbars"))
vbox = gtk.VBox()
iniciar = gtk.Button("Ejecutar Tarea en Archivo")
iniciar.connect("clicked", self.__emit_accion)
vbox.pack_start(iniciar, False, False, 2)
iniciar.set_sensitive(False)
iniciar = gtk.Button("Ejecutar Tareas en la Lista")
iniciar.connect("clicked", self.__emit_accion)
vbox.pack_start(iniciar, False, False, 2)
iniciar.set_sensitive(False)
copiar = gtk.Button("Copiar Tarea a Toda la Lista")
copiar.connect("clicked", self.__emit_accion)
vbox.pack_start(copiar, False, False, 2)
copiar.set_sensitive(False)
eliminar = gtk.Button("Eliminar Tarea")
eliminar.connect("clicked", self.__emit_accion)
vbox.pack_start(eliminar, False, False, 2)
eliminar = gtk.Button("Eliminar Todas las Tareas")
eliminar.connect("clicked", self.__emit_accion)
vbox.pack_start(eliminar, False, False, 2)
self.frame_info = gtk.Frame()
self.frame_info.set_label(" Progreso ")
self.frame_info.set_border_width(5)
self.frame_info.modify_bg(0, get_colors("toolbars"))
self.progress = ProgressBar()
self.frame_info.add(self.progress)
vbox.pack_end(self.frame_info, False, False, 2)
event = gtk.EventBox()
event.set_border_width(5)
event.modify_bg(0, get_colors("windows"))
event.add(vbox)
self.add(event)
self.show_all()
def __init__(self, title, indeterminate=False, gui=False):
self.indeterminate = indeterminate
osxgui = False
self.bar = None
if sys.platform == 'darwin':
osxgui = gui
if osxgui:
try:
self.bar = ProgressBar(title=title,
indeterminate=indeterminate)
except:
pass
# oh well, no CocoaDialog probably
self.reset(title, indeterminate=indeterminate)
self.update('', 0) # Build progress bar string
def process_variants(self, variants):
"""NAME: process_variants
INPUT: variants (list of dictionary with the information needed for submission)
PURPOSE: process all variants classified by the lab"""
progress = ProgressBar(len(variants))
progress.get_next(0)
for i, variant in enumerate(variants):
self.process_variant(variant, self.variantInfo)
self.process_variant(variant, self.expEvidenceInfo)
progress.get_next(i+1)
progress.get_next(len(variants))
print(progress.get_done_message('min'))
def parse_main_page(url):
# r = requests.get(url, headers=setHeader(), cookies=cookies, proxies=proxies)
r = getContent(url)
# print(r.status_code)
soup = bs(r.text, 'html5lib')
videoPages = set()
for link in soup.find_all('a'):
# print(link.get('href'))
ss = link.get('href')
if ss:
if len(ss) > 8 and ss.find('view_video') != -1:
videoPages.add(ss)
print(videoPages)
videoLinks = set()
for link in videoPages:
# page = requests.get(link, headers=setHeader(), cookies=cookies)
page = getContent(link)
# print(page.text.encode(page.encoding).decode('utf-8'))
utext = page.text.encode(page.encoding).decode('utf-8')
soup2 = bs(utext, 'html5lib')
# print(soup2.text)
vurl = soup2.find('video').find('source').get('src')
videoTitle = soup2.find(id='viewvideo-title').get_text().strip()
fileType = re.findall('\.(.{3}?)\?', vurl) # .mp4\.avi
# print(soup2.find('video').find('source').get('src'))
print(vurl)
fileName = videoTitle + '.' + fileType[0]
# print(fileName)
# exit()
# res = requests.get(vurl,stream=True)
# res = getContent(vurl,stream=True)
with closing(getContent(vurl, stream=True)) as res:
chunk_size = 1024
content_size = int(res.headers['content-length'])
progress = ProgressBar(videoTitle,
total=content_size,
unit="KB",
chunk_size=chunk_size,
run_status="正在下载",
fin_status="下载完成")
file = open(fileName, 'wb')
for chunk in res.iter_content(chunk_size=512):
if chunk:
file.write(chunk)
progress.refresh(count=len(chunk))
def createUI(self, oink_widget_list):
self.label = QtGui.QLabel("Choose A Widget:")
self.combo_box_widgets = QtGui.QComboBox()
self.combo_box_widgets.addItems(oink_widget_list)
self.button = QtGui.QPushButton("Launch")
final_layout = QtGui.QHBoxLayout()
final_layout.addWidget(self.label)
final_layout.addWidget(self.combo_box_widgets)
final_layout.addWidget(self.button)
final_page = QtGui.QWidget()
final_page.setLayout(final_layout)
self.progress_bar = ProgressBar()
self.message = QtGui.QLabel("Loading.....")
loading_layout = QtGui.QVBoxLayout()
loading_layout.addWidget(self.progress_bar)
loading_layout.addWidget(self.message)
loading_page = QtGui.QWidget()
loading_page.setLayout(loading_layout)
self.stacked_widget = QtGui.QStackedWidget()
self.stacked_widget.addWidget(final_page)
self.stacked_widget.addWidget(loading_page)
layout = QtGui.QVBoxLayout()
layout.addWidget(self.stacked_widget)
layout.addWidget(Taunter())
self.setLayout(layout)
self.setWindowTitle("OINK Widget Chooser")
icon_file_name_path = os.path.join(MOSES.getPathToImages(),'PORK_Icon.png')
self.setWindowIcon(QtGui.QIcon(icon_file_name_path))
self.show()
def curateTygem(kifuFolder, indexFolder, movesPerGame = 1, totalMoves = 1, previousStates = 3):
movesPerFile = 10000
outFolder = 'outData/'
outfilename = outFolder + input("Enter output file name: ")
storage = Storage(outfilename, movesPerFile, previousStates)
loader = FileLoader(kifuFolder, indexFolder)
fileIndex = 0
movesProcessed = 0
startTime = time.time()
bar = ProgressBar(totalMoves, width=60, fmt=ProgressBar.FULL)
# TODO: Removed Handicap Games!!!!
while movesProcessed < totalMoves:
info, game = loader.next()
mc = processGame(storage, info, game, movesPerGame, previousStates)
movesProcessed += mc
bar(mc)
storage.writeToFile()
endTime = time.time()
print('\nTotal time for ' + str(totalMoves) + ' moves: ', startTime-endTime)
def match(cm, l_cascade, restrictive=0.8):
l_y = len(cm.y_axis)
pb = ProgressBar(len(cm.x_axis), eta_every=5)
for a in range(len(teams[0])):
l_x = len(cm.x_axis)
team = teams[0].pop(0)
if l_x == 0:
teams[0].append(team)
print('')
return False, l_cascade
if l_x < team.people:
teams[0].append(team)
continue
sa1, sa2, sa3 = None, None, None
if team.people == 2:
p1, sa1 = 0, -1
elif team.people == 3:
p1, sa1, sa2 = 0, -1, 1
else:
p1, sa1, sa2, sa3 = 0, -1, 1, -2
popper = [x for x in [p1, sa1, sa2, sa3] if x is not None]
pb.numerator += len(popper)
print("{:>4}".format(restrictive) + str(pb), end='\r')
team.pizzas = cm.pop_xl(popper)
teams[1].append(team)
print('')
if len(teams[0]) == 0:
return False, l_cascade
if len(cm.x_axis) < min(teams[0], key=lambda x: x.people).people:
return False, l_cascade
return True, l_cascade
def Test(self, days, stop, progress=True):
data = fetch_data(timedelta(days=days), stop)
current = get_current()
total_current = []
total_past = []
for i in range(len(data) - self.model.n_inputs):
self.step(data[i:i + self.model.n_inputs])
# total_current.append(self.get_score(current))
# total_past.append(self.get_score(data[i+self.model.n_inputs]))
if progress:
ProgressBar.printProgressBar(
i + 1,
len(data) - self.model.n_inputs,
prefix=f'Testing: {i+1}/{len(data)-self.model.n_inputs}',
length=50)
print(f'Final score: {self.get_score(current)}')
def __init__(self):
w, h = director.get_window_size()
super(ScoreLayer, self).__init__()
# 透明层
self.add(ColorLayer(100, 100, 200, 100, width=w, height=48), z=-1)
self.position = (0, h - 48)
progress_bar = self.progress_bar = ProgressBar(width=200, height=20)
progress_bar.position = 20, 15
self.add(progress_bar)
self.score = Label('Score:', font_size=36,
font_name='Edit Undo Line BRK',
color=(255, 255, 255, 255),
anchor_x='left',
anchor_y='bottom')
self.score.position = (0, 0)
# self.add(self.score)
self.lvl = Label('Lvl:', font_size=36,
font_name='Edit Undo Line BRK',
color=(255, 255, 255, 255),
anchor_x='left',
anchor_y='bottom')
self.lvl.position = (450, 0)
# self.add(self.lvl)
self.objectives_list = []
self.objectives_labels = []
def createUI(self):
self.process_one_button = QtGui.QPushButton("Process Random FSN\nfrom Queue")
self.start_progress_button = QtGui.QPushButton("Start Processing All!")
self.start_progress_button.setFixedSize(200,50)
buttons_stylesheet = """QPushButton{background-color: #66CD00; color: white}; QPushButton::hover{background-color:#cccce5; color: black}"""
self.start_progress_button.setStyleSheet(buttons_stylesheet)
self.image_viewer_widget = ImageViewerWidget()
self.progress_bar = ProgressBar()
self.live_progress = QtGui.QWidget()
self.status_message = QtGui.QLabel("The Cake is a Lie.")
buttons_layout = QtGui.QHBoxLayout()
buttons_layout.addWidget(self.process_one_button,0, QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter)
buttons_layout.addWidget(self.start_progress_button,0, QtCore.Qt.AlignLeft | QtCore.Qt.AlignVCenter)
live_progress_layout = QtGui.QVBoxLayout()
live_progress_layout.addLayout(buttons_layout, 0)
live_progress_layout.addWidget(self.image_viewer_widget,2,QtCore.Qt.AlignHCenter | QtCore.Qt.AlignVCenter)
live_progress_layout.addSpacing(10)
live_progress_layout.addWidget(self.progress_bar,0)
live_progress_layout.addWidget(self.status_message,0,QtCore.Qt.AlignLeft | QtCore.Qt.AlignVCenter)
self.live_progress.setLayout(live_progress_layout)
self.tabs = QtGui.QTabWidget()
self.tabs.addTab(self.live_progress,"Live Progress")
self.log = QtGui.QTextEdit()
self.log.setReadOnly(True)
self.tabs.addTab(self.log,"Log")
layout = QtGui.QHBoxLayout()
layout.addWidget(self.tabs)
self.setLayout(layout)
self.show()
def __init__(self):
w, h = director.get_window_size()
super(ScoreLayer, self).__init__()
# transparent layer
trblue = c.blue[0:3] + (100, ) # <- transparency
self.add(ColorLayer(*trblue, width=w, height=SCORE_H), z=-1)
self.position = (0, h - SCORE_H)
progress_bar = self.progress_bar = ProgressBar(width=PROG_W,
height=PROG_H)
dist = (SCORE_H - PROG_H) // 2
progress_bar.position = dist, dist
self.add(progress_bar)
labelSettings = dict(font_size=f.size_selected,
font_name=f.default,
color=c.white,
anchor_x='right',
anchor_y='center')
self.score = Label('Score:', **labelSettings)
self.score.position = (w - dist, SCORE_H // 2)
self.add(self.score)
#lvl
labelSettings['anchor_x'] = 'center'
self.lvl = Label('Lvl:', **labelSettings)
self.lvl.position = (w // 2, SCORE_H // 2)
self.add(self.lvl)
def progressIter( fn, lst ):
progress = ProgressBar( 0, len(lst), mode='fixed' )
oldprog = str(progress)
for n in lst:
v = fn( n )
if v:
progress.update_amount(v)
else:
progress.increment_amount()
if oldprog != str(progress):
print progress, "\r",
sys.stdout.flush()
oldprog=str(progress)
print '\n'
def reset(self, title, indeterminate=False):
if indeterminate != self.indeterminate:
if self.bar: self.bar.finish()
self.indeterminate = indeterminate
if self.bar:
self.bar = ProgressBar(title=title, indeterminate=indeterminate)
self.progBar = "[]" # This holds the progress bar string
self.width = 40
self.amount = 0 # When amount == max, we are 100% done
def progressMap( fn, lst ):
progress = ProgressBar( 0, len(lst), mode='fixed' )
oldprog = str(progress)
out = []
for n in lst:
v = fn( n )
out.append( v )
progress.increment_amount()
if oldprog != str(progress):
print progress, "\r",
sys.stdout.flush()
oldprog=str(progress)
print '\n'
return out
def PrintSiteReadinessMetricsStats(self):
print "\nPrinting Site Readiness Metrics Statistics\n"
prog = ProgressBar(0, 100, 77)
fileHandle = open(self.fileReadinessStat , 'w')
sites = self.matrices.readiValues.keys()
sites.sort()
for sitename in sites:
dates = self.matrices.readiValues[sitename].keys()
dates.sort()
continue
for i in "T1","T2":
prog.increment(100./2.)
for dat in dates:
countR=0; countW=0; countNR=0; countSD=0; countna=0
for sitename in sites:
if sitename.find("T1_CH_CERN") == 0: continue
if not sitename.find(i+"_") == 0: continue
if self.SkipSiteOutput(sitename): continue
state = self.matrices.readiValues[sitename][dat]
if state == "R":
countR+=1
if state == "W":
countW+=1
if state == "NR":
countNR+=1
if state.find("n/a") == 0:
countna+=1
if state == "SD":
countSD+=1
if dat == self.tinfo.todaystamp: continue
tofile = "Site Readiness Metric " + i + " " + dat + " " + str(countR) + " " + str(countNR) + " " + str(countna) + " " + str(countW) + " " + str(countSD) + " " + str(countR+countNR+countW+countna+countSD) + "\n"
fileHandle.write(tofile)
fileHandle.close()
prog.finish()
def EvaluateDailyMetric(self):
print "\nEvaluating Daily Status\n"
# set value for the 'Daily Metric' column in self.matrices.dailyMetrics
prog = ProgressBar(0, 100, 77)
for sitename in self.matrices.columnValues:
prog.increment(100./len(self.matrices.columnValues))
self.matrices.dailyMetrics[sitename] = {}
items = self.matrices.columnValues[sitename].keys()
items.sort()
status = ' '
for day in items:
status = 'O'
for crit in self.GetCriteriasList(sitename): # loop through the columns (criteria) that apply to this site
if not self.matrices.columnValues[sitename][day].has_key(crit):
info = {}
info['Status'] = 'n/a'
info['Color'] = 'white'
self.matrices.columnValues[sitename][day][crit] = info
if self.matrices.columnValues[sitename][day][crit]['Color'] == 'red':
status = 'E'
if self.matrices.columnValues[sitename][day]['Downtimes_top']['Color'] == 'brown':
status = 'SD'
# exclude sites that are not in SiteDB
testdate = date(int(day[0:4]),int(day[5:7]),int(day[8:10]))
sitedbtimeint = testdate - date(2009,11,03) # magic number - no idea where it comes from.
if sitedbtimeint.days >= 0:
if self.matrices.columnValues[sitename][day].has_key('IsSiteInSiteDB'):
if self.matrices.columnValues[sitename][day]['IsSiteInSiteDB']['Color'] == 'white':
status = 'n/a'
if day == self.tinfo.todaystamp:
status = ' '
self.matrices.dailyMetrics[sitename][day] = status
prog.finish()
def __init__(self, title, indeterminate=False, gui=False):
self.indeterminate = indeterminate
osxgui = False
self.bar = None
if sys.platform == 'darwin':
osxgui = gui
if osxgui:
try:
self.bar = ProgressBar(title=title, indeterminate=indeterminate)
except:
pass
# oh well, no CocoaDialog probably
self.reset(title, indeterminate=indeterminate)
self.update('', 0) # Build progress bar string
def __init__(self):
gtk.Frame.__init__(self)
self.set_label(" Acciones: ")
self.set_border_width(5)
self.modify_bg(0, get_colors("toolbars"))
vbox = gtk.VBox()
iniciar = gtk.Button("Ejecutar Tarea en Archivo")
iniciar.connect("clicked", self.__emit_accion)
vbox.pack_start(iniciar, False, False, 2)
iniciar.set_sensitive(False)
iniciar = gtk.Button("Ejecutar Tareas en la Lista")
iniciar.connect("clicked", self.__emit_accion)
vbox.pack_start(iniciar, False, False, 2)
iniciar.set_sensitive(False)
copiar = gtk.Button("Copiar Tarea a Toda la Lista")
copiar.connect("clicked", self.__emit_accion)
vbox.pack_start(copiar, False, False, 2)
copiar.set_sensitive(False)
eliminar = gtk.Button("Eliminar Tarea")
eliminar.connect("clicked", self.__emit_accion)
vbox.pack_start(eliminar, False, False, 2)
eliminar = gtk.Button("Eliminar Todas las Tareas")
eliminar.connect("clicked", self.__emit_accion)
vbox.pack_start(eliminar, False, False, 2)
self.frame_info = gtk.Frame()
self.frame_info.set_label(" Progreso ")
self.frame_info.set_border_width(5)
self.frame_info.modify_bg(0, get_colors("toolbars"))
self.progress = ProgressBar()
self.frame_info.add(self.progress)
vbox.pack_end(self.frame_info, False, False, 2)
event = gtk.EventBox()
event.set_border_width(5)
event.modify_bg(0, get_colors("windows"))
event.add(vbox)
self.add(event)
self.show_all()
def EvaluateDailyMetric(self):
print "\nEvaluating Daily Metric\n"
# set value for the 'Daily Metric' column in self.matrices.dailyMetrics
# NOTE: also sets n/a in columnValues for missing metrics
prog = ProgressBar(0, 100, 77)
for sitename in self.matrices.columnValues:
prog.increment(100./len(self.matrices.columnValues))
self.matrices.dailyMetrics[sitename] = {}
items = self.matrices.columnValues[sitename].keys()
items.sort()
status = ' '
for day in items:
status = 'O' # initial value is OK ('O')
for crit in self.GetCriteriasList(sitename): # loop through the columns (criteria) that apply to this site and affect site status
if not self.matrices.columnValues[sitename][day].has_key(crit): # fill columnValues with 'n/a' for any missing values
self.matrices.columnValues[sitename][day][crit] = self.nullInfo()
if self.matrices.columnValues[sitename][day][crit]['Color'] == 'red': # if any individual metric is red, set status to error ('E')
status = 'E'
if self.matrices.columnValues[sitename][day]['Downtimes_top']['Color'] == 'brown': # if site was in downtime set to 'SD'
status = 'SD'
# exclude sites that are not in SiteDB
testdate = date(int(day[0:4]),int(day[5:7]),int(day[8:10]))
sitedbtimeint = testdate - date(2009,11,03) # magic number - no idea where it comes from.
if sitedbtimeint.days >= 0:
if self.matrices.columnValues[sitename][day].has_key('IsSiteInSiteDB'):
if self.matrices.columnValues[sitename][day]['IsSiteInSiteDB']['Color'] == 'white':
status = 'n/a'
#status = status
if day == self.tinfo.todaystamp: # set today's to the blank character
status = ' '
self.matrices.dailyMetrics[sitename][day] = status
prog.finish()
def createUI(self):
#self.raw_data_filter_form = FilterForm()
self.fsn_entry_field = FSNTextEdit()
self.raw_data_table = CopiableQTableWidget(0,0)
self.upload_raw_data_button = QtGui.QPushButton("Upload Raw Data from File")
self.progress_bar = ProgressBar()
self.progress_log = QtGui.QTextEdit()
layout = QtGui.QVBoxLayout()
layout.addWidget(self.upload_raw_data_button)
layout.addWidget(self.progress_bar)
layout.addWidget(self.progress_log)
self.setLayout(layout)
self.setWindowTitle("Raw Data Uploader")
self.setWindowIcon(QtGui.QIcon(os.path.join(MOSES.getPathToImages(),"PORK_Icon.png")))
self.show()
class progressBar:
def __init__(self, title, indeterminate=False, gui=False):
self.indeterminate = indeterminate
osxgui = False
self.bar = None
if sys.platform == 'darwin':
osxgui = gui
if osxgui:
try:
self.bar = ProgressBar(title=title, indeterminate=indeterminate)
except:
pass
# oh well, no CocoaDialog probably
self.reset(title, indeterminate=indeterminate)
self.update('', 0) # Build progress bar string
def reset(self, title, indeterminate=False):
if indeterminate != self.indeterminate:
if self.bar: self.bar.finish()
self.indeterminate = indeterminate
if self.bar:
self.bar = ProgressBar(title=title, indeterminate=indeterminate)
self.progBar = "[]" # This holds the progress bar string
self.width = 40
self.amount = 0 # When amount == max, we are 100% done
def finish(self):
if self.bar: self.bar.finish()
def update(self, message='', fraction=0.0, after_args=()):
self.message = message
# Figure out how many hash bars the percentage should be
allFull = self.width - 2
percentDone = int(round(fraction*100))
numHashes = int(round(fraction * allFull))
# build a progress bar with hashes and spaces
self.progBar = "[" + '#'*numHashes + ' '*(allFull-numHashes) + "]"
# figure out where to put the percentage, roughly centered
percentPlace = (len(self.progBar) / 2) - len(str(percentDone))
percentString = ' '+ str(percentDone) + r"% "
# slice the percentage into the bar
self.progBar = self.progBar[0:percentPlace] + percentString + \
self.progBar[percentPlace+len(percentString):]
if self.bar: self.bar.update(percentDone, message)
if self.indeterminate:
sys.stdout.write(self.message + ' '.join(after_args) + '\r')
else:
sys.stdout.write(self.message + str(self.progBar) + ' '.join(after_args) + '\r')
sys.stdout.flush()
def __str__(self):
return str(self.progBar)
def write_heuristics_data(start, n, res_fname, states_fname='data/states.txt', search=a_star, h_name='linear_conflict', heuristic=NPuzzle.linear_conflict):
"""
Write a file with information on states, depths, expanded nodes, and
running time for the 3 heuristics implemented on a specific search method.
"""
from ProgressBar import ProgressBar
import time
import re
with open(res_fname, 'a') as res_f, open(states_fname, 'r') as s_f:
K = 3
goal = NPuzzle(K, range(K*K))
for i in range(start):
s_f.readline()
print 'Reading states from file {:s}'.format(states_fname)
print 'Writing {} states data to file {:s}'.format(n, res_fname)
pb = ProgressBar() # a simple pogressbar
pb.start()
f_format = '{};{};{};{}\n'
if start == 0:
columns = ['state', 'steps', h_name + '_nodes', h_name + '_time']
res_f.write(f_format.format(*columns))
for i in range(n - start):
state_str = s_f.readline().strip()
state = [int (b) for b in re.findall('[0-9]+', state_str)]
initial = NPuzzle(3, state)
percent = float(i+start)/(n)
pb.update(percent)
try:
init_time1 = time.time()
n1, s1, path1 = search(initial, goal, heuristic)
end_time1 = time.time()
t1 = end_time1 - init_time1
except KeyboardInterrupt:
break
except:
t1 = 'NaN'
n1 = 'NaN'
s1 = 'NaN'
res_f.write(f_format.format(initial, s1, n1, t1))
res_f.flush()
pb.finish()
def createUI(self):
path_to_image = os.path.join(MOSES.getPathToImages(), "OINK.png")
self.image = ImageLabel(path_to_image, 800, 566)
#self.sharingan = SharinganButton()
self.progress_bar = ProgressBar()
self.message = QtGui.QLabel("Not all those who wander are lost....")
layout = QtGui.QVBoxLayout()
layout.addWidget(self.image)
layout.addWidget(self.progress_bar)
layout.addWidget(self.message)
layout.addWidget(Taunter())
self.setLayout(layout)
self.setWindowTitle("Bulk Buster: The OINK Preloader Screen")
icon_file_name_path = os.path.join(MOSES.getPathToImages(),'PORK_Icon.png')
self.setWindowIcon(QtGui.QIcon(icon_file_name_path))
self.show()
def createUI(self):
self.fsns_label = QtGui.QLabel("FSNs\Item IDs:")
self.fsns_text_edit = QtGui.QTextEdit()
self.fsns_text_edit.setToolTip("Paste a list of FSNs or Item IDs here,\nseparated either by a new line or a comma.")
self.type_selector = QtGui.QComboBox()
self.type_selector.addItems(["FSN(s)", "Item ID(s)"])
self.type_selector.setToolTip("Select the list type. Are you searching by FSNs or Item IDs?")
self.type_selector.setCurrentIndex(0)
self.output_table = CopiableQTableWidget(0, 0)
self.progress_bar = ProgressBar()
self.fetch_data_button = ImageButton(os.path.join("Images","find.png"),64,64,os.path.join("Images","find_mouseover.png"))
self.fetch_data_button.setFlat(True)
form_searcher_layout = QtGui.QVBoxLayout()
form_searcher_layout.addWidget(self.fsns_label, 0)
form_searcher_layout.addWidget(self.fsns_text_edit, 2)
self.seeker_button = ImageButton(os.path.join("Images","seeker.png"),100,100,os.path.join("Images","seeker_mouseover.png"))
self.seeker_button.setFlat(True)
self.seeker_button.setToolTip("You're a Wizard, Harry.")
form_options_layout = QtGui.QVBoxLayout()
form_options_layout.addStretch(1)
form_options_layout.addWidget(self.seeker_button,1, QtCore.Qt.AlignHCenter | QtCore.Qt.AlignBottom)
form_options_layout.addStretch(2)
form_options_layout.addWidget(self.type_selector, 0)
form_options_layout.addWidget(self.fetch_data_button, 0, QtCore.Qt.AlignHCenter | QtCore.Qt.AlignBottom)
form_layout = QtGui.QHBoxLayout()
form_layout.addLayout(form_searcher_layout, 3)
form_layout.addLayout(form_options_layout, 0)
layout = QtGui.QVBoxLayout()
layout.addLayout(form_layout, 0)
layout.addWidget(self.output_table, 3)
layout.addWidget(self.progress_bar, 0)
self.setLayout(layout)
self.setWindowTitle("Seeker: The FSN Finding Tool")
self.setWindowIcon(QtGui.QIcon(os.path.join('Images','PORK_Icon.png')))
self.show()
def ProduceSiteReadinessRankingPlots(self):
print "\nProducing Site Readiness Ranking plots\n"
prog = ProgressBar(0, 100, 77)
sitesit = self.matrices.readiValues.keys()
sitesit.sort()
for dayspan in 30, 15, 7:
prog.increment(100./3.)
for pl in 'SD_perc', 'R+Wcorr_perc':
for i in "T1","T2":
dataR = {}
filename = self.plotOutDir + "/" + i + "_" + pl + "_last" + str(dayspan) + "days_" + self.tinfo.timestamphtml + ".png"
for sitename in sitesit:
if not sitename.find(i+"_") == 0 : continue
if self.SkipSiteOutput(sitename): continue
if pl == 'SD_perc' and self.matrices.stats[sitename][dayspan][pl]==0.: continue # Do not show Up sites on SD plots.
if sitename == 'T1_CH_CERN': sitename = 'T2_CH_CERN'
dataR[sitename+" ("+str(self.matrices.stats[sitename][dayspan]["SD_perc"])+"%)"] = self.matrices.stats[sitename][dayspan][pl]
if len(dataR) == 0:
os.system("touch %s" % filename)
continue
norms = normalize(0,100)
mapper = cm.ScalarMappable( cmap=cm.RdYlGn, norm=norms )
# Hack to make mapper work:
def get_alpha(*args, **kw):
return 1.0
mapper.get_alpha = get_alpha
A = linspace(0,100,100)
mapper.set_array(A)
pos = arange(len(dataR))+.5 # the bar centers on the y axis
dataS = dataR.items()
dataS.sort(lambda x,y: cmp(x[1],y[1]))
ytext = []
val = []
color = []
total=0
ent=0
ent2=0
for t in range(0,len(dataS)):
ytext.append(dataS[t][0])
val.append(dataS[t][1])
color.append( mapper.to_rgba( dataS[t][1] ) )
total += 1
if i == 'T1' and dataS[t][1] <= 90 : ent+=1
if i == 'T1' and dataS[t][1] > 90 : ent2+=1
if i == 'T2' and dataS[t][1] <= 80 : ent+=1
if i == 'T2' and dataS[t][1] > 80 : ent2+=1
if pl == 'R+Wcorr_perc':
metadataR = {'title':'%s Readiness Rank last %i days (+SD %%) [%s]' % (i,int(dayspan),self.tinfo.todaystamp), 'fixed-height':False }
if pl == 'SD_perc':
metadataR = {'title':'Rank for %s Scheduled Downtimes last %i days [%s]' % (i,int(dayspan),self.tinfo.todaystamp), 'fixed-height':True}
fig = Figure()
canvas = FigureCanvas(fig)
if i == 'T1':
SRlim=90
fig.set_size_inches(7,4)
else:
SRlim=80
fig.set_size_inches(7.5,9.5)
ax = fig.add_subplot(111)
fig.subplots_adjust(left=0.2, right=0.97)
ax.set_autoscale_on(False)
ax.barh(pos,val, align='center',color=color)
ax.set_xlim([0,100])
ax.set_ylim([0,len(dataS)])
ax.set_yticklabels(ytext,fontsize=8,family='sans-serif')
ax.set_yticks(pos)
ax.set_title(metadataR['title'],fontsize=14)
ax.set_xlabel('Site Readiness %',fontsize=12,family='sans-serif')
if pl == 'R+Wcorr_perc':
ax.axvline(x=SRlim, ymin=0, ymax=1,color='red',ls=':',lw=3)
if i == 'T1' :
ax.text(91,0.65,str(ent2)+"/"+str(total)+" >90%",color='darkgreen',fontsize=6)
ax.text(91,0.3,str(ent)+"/"+str(total)+" $\leq$90%",color='red',fontsize=6)
if i == 'T2' :
ax.text(81,2,str(ent2)+"/"+str(total)+" >80%",color='darkgreen',fontsize=6)
ax.text(81,1,str(ent)+"/"+str(total)+" $\leq$80%",color='red',fontsize=6)
canvas.print_figure(filename)
prog.finish()
def ProduceSiteReadinessHTMLViews(self):
print "\nProducing Site Readiness HTML view\n"
colspans1 = str(self.cinfo.daysToShow+1)
colspans2 = str(self.cinfo.daysToShow+1)
colspans22 = str(self.cinfo.daysToShow+2)
colspans3 = str(self.cinfo.daysSC)
colspans4 = str(self.cinfo.daysSC)
colspans5 = str(self.cinfo.daysToShow-self.cinfo.daysSC)
dw = 45
mw = 325
tablew = str((self.cinfo.daysToShow)*dw+mw)
dayw = str(dw)
metricw = str(mw)
daysw = str((self.cinfo.daysToShow)*dw)
scdaysw1 = str((self.cinfo.daysSC)*dw)
scdaysw = str((self.cinfo.daysSC)*dw)
filehtml = self.htmlOutDir + '/SiteReadinessReport_' + self.tinfo.timestamphtml +'.html'
fileHandle = open ( filehtml , 'w' )
fileHandle.write("<html><head><title>CMS Site Readiness</title><link type=\"text/css\" rel=\"stylesheet\" href=\""+self.options.css+"/style-css-reports.css\"/></head>\n")
fileHandle.write("<body><center>\n")
fileHandle.write('<h1 style="line-height:200%">Site Readiness Report</h1>')
sitesit = self.matrices.readiValues.keys()
sitesit.sort()
prog = ProgressBar(0, 100, 77)
for sitename in sitesit:
prog.increment(100./len(sitesit))
if not self.SkipSiteOutput(sitename):
fileHandle.write("<a name=\""+ sitename + "\"></a>\n\n")
fileHandle.write("<div id=para-"+ sitename +">\n")
fileHandle.write("<table border=\"0\" cellspacing=\"0\" class=stat>\n")
fileHandle.write("<tr height=4><td width=" + metricw + "></td>\n")
fileHandle.write("<td width=" + daysw + " colspan=" + colspans1 + " bgcolor=black></td></tr>\n")
fileHandle.write("<tr>\n")
fileHandle.write("<td width=\"" + metricw + "\"></td>\n")
fileHandle.write("<td width=\"" + daysw + "\" colspan=" + colspans1 + " bgcolor=darkblue><div id=\"site\">" + sitename + "</div></td>\n")
fileHandle.write("</tr>\n")
fileHandle.write("<tr height=4><td width=" + metricw + "></td>\n")
fileHandle.write("<td width=" + daysw + " colspan=" + colspans1 + " bgcolor=black></td></tr>\n")
fileHandle.write("<tr height=7><td width=" + metricw + "></td>\n")
fileHandle.write("<td width=" + daysw + " colspan=" + colspans1 + "></td></tr>\n")
dates = self.matrices.dailyMetrics[sitename].keys()
dates.sort()
fileHandle.write("<tr height=4><td width=" + metricw + "></td>\n")
fileHandle.write("<td width=" + daysw + " colspan=" + colspans1 + " bgcolor=black></td></tr>\n")
fileHandle.write("<tr><td width=" + metricw + "></td>\n")
igdays=0
indmetrics = self.cinfo.metorder.keys()
indmetrics.sort()
for metnumber in indmetrics:
met = self.cinfo.metorder[metnumber] #colName
met1 = self.cinfo.printCol[met] #pCol (print permission)
tier = sitename.split("_")[0]
met2 = self.cinfo.criteria[tier]
dividMetrics = '"metrics2"'
if met == 'LifeStatus' or met == 'SiteReadiness':
dividMetrics = '"daily-metric"'
#if not self.matrices.columnValues[sitename][dates[0]].has_key(met) or met == 'IsSiteInSiteDB': continue # ignore
if not self.matrices.columnValues[sitename][dates[0]].has_key(met) or met1 == '0' : continue # ignore
if met1 == 't' and not met in met2 : continue # ignore
if sitename.find("T1_CH_CERN") == 0 and met == 'T1linksfromT0': continue # ignore
if met == 'SAMAvailability':
fileHandle.write("<tr><td width=\"" + metricw + "\"><div id=\"metrics-header\"><font color=\"orange\">" + self.cinfo.metlegends[met] + ": </font></div></td>\n")
elif met == 'LifeStatus':
fileHandle.write("<tr><td width=\"" + metricw + "\"><div id=\"metrics-header\">" + self.cinfo.metlegends[met] + ": </div></td><td width=\"" + dayw + "\" bgcolor=white>"+"</td>\n")
else:
fileHandle.write("<tr><td width=\"" + metricw + "\"><div id=\"metrics-header\">" + self.cinfo.metlegends[met] + ": </div></td>\n")
igdays=0
for datesgm in dates: # write out a line for each constituent metric
igdays+=1
if (self.cinfo.days - igdays)>self.cinfo.daysToShow-1: continue
state = self.matrices.columnValues[sitename][datesgm][met]['Status']
colorst=self.matrices.columnValues[sitename][datesgm][met]['Color']
datesgm1 = datesgm[8:10]
c = datetime.datetime(*time.strptime(datesgm,"%Y-%m-%d")[0:5])
if not sitename in self.weekendSites and (c.weekday() == 5 or c.weekday() == 6) and sitename.find('T2_') == 0 and met != "LifeStatus": # id. weekends
if state != " " :
if self.matrices.columnValues[sitename][datesgm][met].has_key('URL') and self.matrices.columnValues[sitename][datesgm][met]['URL'] != ' ' :
stateurl=self.matrices.columnValues[sitename][datesgm][met]['URL']
fileHandle.write("<td width=\"" + dayw + "\" bgcolor=grey><a href=\""+stateurl+"\">"+"<div id="+dividMetrics+">" + state + "</div></a></td>\n")
else:
fileHandle.write("<td width=\"" + dayw + "\" bgcolor=grey><div id="+dividMetrics+">" + state + "</div></td>\n")
else:
fileHandle.write("<td width=\"" + dayw + "\" bgcolor=white><div id="+dividMetrics+">" + state + "</div></td>\n")
else:
if self.matrices.columnValues[sitename][datesgm][met].has_key('URL') and self.matrices.columnValues[sitename][datesgm][met]['URL'] != ' ' :
stateurl=self.matrices.columnValues[sitename][datesgm][met]['URL']
fileHandle.write("<td width=\"" + dayw + "\" bgcolor=" + colorst + "><a href=\""+stateurl+"\">"+"<div id="+dividMetrics+">" + state + "</div></a></td>\n")
else:
fileHandle.write("<td width=\"" + dayw + "\" bgcolor=" + colorst + "><div id="+dividMetrics+">" + state + "</div></td>\n")
fileHandle.write("</tr>\n")
if met == 'SiteReadiness':
fileHandle.write("<tr height=4><td width=" + tablew + " colspan=" + colspans22 + " bgcolor=black></td></tr>\n")
fileHandle.write("<tr height=2><td width=" + tablew + " colspan=" + colspans22 + " bgcolor=white></td></tr>\n")
fileHandle.write("<tr height=4><td width=" + tablew + " colspan=" + colspans22 + " bgcolor=black></td></tr>\n")
fileHandle.write("<tr height=4><td width=" + metricw + "></td>\n")
igdays=0
for datesgm in dates:
igdays+=1
if (self.cinfo.days - igdays)>self.cinfo.daysToShow-1: continue
datesgm1 = datesgm[8:10]
c = datetime.datetime(*time.strptime(datesgm,"%Y-%m-%d")[0:5])
if c.weekday() == 5 or c.weekday() == 6: # id. weekends
fileHandle.write("<td width=" + dayw + " bgcolor=grey> <div id=\"date\">" + datesgm1 + "</div></td>\n")
else:
fileHandle.write("<td width=" + dayw + " bgcolor=lightgrey> <div id=\"date\">" + datesgm1 + "</div></td>\n")
fileHandle.write("</tr>\n")
fileHandle.write("<tr height=4><td width=" + metricw + "></td>\n")
fileHandle.write("<td width=" + daysw + " colspan=" + colspans1 + " bgcolor=black></td></tr>\n")
fileHandle.write("<tr><td width=" + metricw + "></td>\n")
lastmonth=""
igdays=0
for datesgm in dates:
igdays+=1
if (self.cinfo.days - igdays)>self.cinfo.daysToShow-1: continue
c = datetime.datetime(*time.strptime(datesgm,"%Y-%m-%d")[0:5])
month = c.strftime("%b")
if month != lastmonth:
fileHandle.write("<td width=" + dayw + " bgcolor=black> <div id=\"month\">" + month + "</div></td>\n")
lastmonth=month
else:
fileHandle.write("<td width=" + dayw + "></td>\n")
fileHandle.write("</tr>\n")
fileHandle.write("<tr><td width=" + metricw + "></td>\n")
fileHandle.write("<td width=" + scdaysw1 + " colspan=" + colspans3 + "></td>\n")
fileHandle.write("</table>\n")
# report time
fileHandle.write("<div id=\"leg1\">" + self.reptime + "</div>\n")
fileHandle.write("</div>\n")
# print instructions
if sitename.find('T1_') == 0:
fileHandle.write(self.t1Inst)
elif sitename.find('T2_') == 0:
fileHandle.write(self.t2Inst)
fileHandle.write("<p>\n")
fileHandle.write("<p><br>\n")
fileHandle.write("</center></html></body>")
fileHandle.close()
prog.finish()
def createUI(self):
self.group_box = QtGui.QGroupBox("Data Selector")
self.page_selector = IconListBox()
page_control_list = [
{
"Name": "From Flipkart Using FSNs",
"Icon": os.path.join("essentials","download.png")
},
{
"Name": "From CSV Data File",
"Icon": os.path.join("essentials","csv_file.png")
}
]
self.page_selector.addElements(page_control_list)
self.page_selector.setFixedSize(302,110)
#FSN Mode Widget
self.fsn_mode_widget = QtGui.QGroupBox("Data By FSN")
self.fsn_text_edit = FSNTextEdit()
self.fsn_text_edit.setFixedSize(450,400)
self.category_label = QtGui.QLabel("Category:")
self.category_combo_box = QtGui.QComboBox()
self.category_combo_box.addItems(getCategoryFolderNames()) #Later, add this data from OINK's server.
self.category_combo_box.setToolTip("Select the default category for the given FSNs.\nNote that mixing various types of FSNs isn't recommended.\nThe icons won't load.")
self.attributes_list_box = QtGui.QListWidget()
self.attributes_list_box.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection)
self.attributes_list_box.setToolTip("Displays all product attributes, obtained from the FK server.")
self.primary_attributes_list_box = QtGui.QListWidget()
self.primary_attributes_list_box.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection)
self.primary_attributes_list_box.setToolTip("Displays primary product attributes that you have selected.")
self.secondary_attributes_list_box = QtGui.QListWidget()
self.secondary_attributes_list_box.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection)
self.secondary_attributes_list_box.setToolTip("Displays secondary product attributes that you have selected.")
self.push_to_primary_button = QtGui.QPushButton("Add to\nPrimary List")
self.push_to_primary_button.setToolTip("Click to move the chosen attribute into the list of primary attributes.")
self.remove_from_primary_button = QtGui.QPushButton("Remove from\nPrimary List")
self.remove_from_primary_button.setToolTip("Click to move the chosen attribute out of the list of primary attributes.")
self.push_to_secondary_button = QtGui.QPushButton("Add to\nSecondary List")
self.push_to_secondary_button.setToolTip("Click to move the chosen attribute into the list of secondary attributes.")
self.remove_from_secondary_button = QtGui.QPushButton("Remove from\nSecondary List")
self.remove_from_secondary_button.setToolTip("Click to move the chosen attribute out of the list of secondary attributes.")
#downloader
self.fetch_images_attributes_button = QtGui.QPushButton("Download")
self.fetch_images_attributes_button.setToolTip("This will check if parent images are available for all the FSNs and download them if necessary from the FK site. It will also load the spec table.")
self.export_scraped_data_button = QtGui.QPushButton("Export Data")
self.fetching_progress = ProgressBar()
self.fetching_progress.setRange(0,100)
self.fetching_progress.setValue(0)
self.fetching_activity = QtGui.QLabel("All that is gold does not glitter!")
self.fetching_activity.setStyleSheet("QLabel{font: 10px black; border: 1px solid black;}")
self.fetching_activity.setToolTip("This indicates the current downloader's activity, or some random quote that Vinay thinks is funny.")
self.completed_fsns_count_label = QtGui.QLabel("Completed:")
self.completed_fsns_count_spinbox = QtGui.QSpinBox()
self.completed_fsns_count_spinbox.setEnabled(False)
self.eta_label = QtGui.QLabel("ETA:")
self.eta_datetimeedit = QtGui.QDateTimeEdit()
self.eta_datetimeedit.setEnabled(False)
self.eta_datetimeedit.setMinimumDateTime(QtCore.QDateTime(datetime.datetime.now()))
self.activity_log_textedit = QtGui.QTextEdit()
self.activity_log_textedit.setReadOnly(True)
self.pending_fsns_count_label = QtGui.QLabel("Pending:")
self.pending_fsns_count_spinbox = QtGui.QSpinBox()
self.pending_fsns_count_spinbox.setEnabled(False)
self.failed_fsns_label = QtGui.QLabel("Failed:")
self.failed_fsns_count_spinbox = QtGui.QSpinBox()
self.failed_fsns_count_spinbox.setEnabled(False)
self.completed_fsns_count_spinbox.setRange(0,99999999)
self.pending_fsns_count_spinbox.setRange(0,99999999)
self.failed_fsns_count_spinbox.setRange(0,99999999)
self.pending_fsns_list_text_edit = QtGui.QTextEdit()
self.pending_fsns_list_text_edit.setReadOnly(True)
self.completed_fsns_list_text_edit = QtGui.QTextEdit()
self.completed_fsns_list_text_edit.setReadOnly(True)
self.failed_fsns_list_text_edit = QtGui.QTextEdit()
self.failed_fsns_list_text_edit.setReadOnly(True)
buttons_and_progress_bar = QtGui.QHBoxLayout()
buttons_and_progress_bar.addWidget(self.fetch_images_attributes_button, 2)
buttons_and_progress_bar.addWidget(self.export_scraped_data_button, 1)
buttons_and_progress_bar.addWidget(self.fetching_progress, 4)
completed_tracking = QtGui.QVBoxLayout()
completed_tracking.addWidget(self.completed_fsns_count_label)
completed_tracking.addWidget(self.completed_fsns_count_spinbox)
completed_tracking.addWidget(self.completed_fsns_list_text_edit)
eta_layout = QtGui.QHBoxLayout()
eta_layout.addWidget(self.eta_label)
eta_layout.addWidget(self.eta_datetimeedit)
pending_tracking = QtGui.QVBoxLayout()
pending_tracking.addWidget(self.pending_fsns_count_label)
pending_tracking.addWidget(self.pending_fsns_count_spinbox)
pending_tracking.addWidget(self.pending_fsns_list_text_edit)
pending_tracking.addLayout(eta_layout)
failed_tracking = QtGui.QVBoxLayout()
failed_tracking.addWidget(self.failed_fsns_label)
failed_tracking.addWidget(self.failed_fsns_count_spinbox)
failed_tracking.addWidget(self.failed_fsns_list_text_edit)
fsns_tracking = QtGui.QHBoxLayout()
fsns_tracking.addLayout(completed_tracking)
fsns_tracking.addLayout(pending_tracking)
fsns_tracking.addLayout(failed_tracking)
downloader_layout = QtGui.QVBoxLayout()
downloader_layout.addLayout(buttons_and_progress_bar)
downloader_layout.addLayout(fsns_tracking)
downloader_layout.addWidget(self.fetching_activity)
downloader = QtGui.QWidget()
downloader.setLayout(downloader_layout)
downloader_tabs = QtGui.QTabWidget()
downloader_tabs.addTab(downloader, "Downloader")
downloader_tabs.addTab(self.activity_log_textedit, "Log")
self.fsn_mode_data_options = QtGui.QGroupBox("Data Options")
fsn_mode_data_options_layout = QtGui.QGridLayout()
fsn_mode_data_options_layout.addWidget(self.category_label,0,0,1,2, QtCore.Qt.AlignVCenter)
fsn_mode_data_options_layout.addWidget(self.category_combo_box,0,2,1,2,QtCore.Qt.AlignVCenter)
fsn_mode_data_options_layout.addWidget(self.attributes_list_box,1,0,4,4, QtCore.Qt.AlignHCenter)
fsn_mode_data_options_layout.addWidget(self.primary_attributes_list_box,1,5,2,2, QtCore.Qt.AlignHCenter)
fsn_mode_data_options_layout.addWidget(self.push_to_primary_button,1,4,1,1, QtCore.Qt.AlignHCenter | QtCore.Qt.AlignVCenter)
fsn_mode_data_options_layout.addWidget(self.remove_from_primary_button,2,4,1,1, QtCore.Qt.AlignHCenter | QtCore.Qt.AlignVCenter)
fsn_mode_data_options_layout.addWidget(self.secondary_attributes_list_box,3,5,2,2, QtCore.Qt.AlignHCenter)
fsn_mode_data_options_layout.addWidget(self.push_to_secondary_button,3,4,1,1, QtCore.Qt.AlignHCenter | QtCore.Qt.AlignVCenter)
fsn_mode_data_options_layout.addWidget(self.remove_from_secondary_button,4,4,1,1, QtCore.Qt.AlignHCenter | QtCore.Qt.AlignVCenter)
self.fsn_mode_data_options.setLayout(fsn_mode_data_options_layout)
self.fsn_mode_data_options.setEnabled(False)
fsn_mode_layout = QtGui.QGridLayout()
fsn_mode_layout.addWidget(self.fsn_text_edit,0,0,7,1)
downloader_tabs
fsn_mode_layout.addWidget(downloader_tabs,0,1,2,5, QtCore.Qt.AlignBottom)
#fsn_mode_layout.addWidget(self.fetch_images_attributes_button,0,1,1,2, QtCore.Qt.AlignBottom)
#fsn_mode_layout.addWidget(self.fetching_progress,0,3,1,5, QtCore.Qt.AlignBottom)
#fsn_mode_layout.addWidget(self.fetching_activity,1,3,1,5, QtCore.Qt.AlignTop)
#fsn_mode_layout.addWidget(self.export_scraped_data_button,1,1,1,2, QtCore.Qt.AlignTop)
fsn_mode_layout.addWidget(self.fsn_mode_data_options,2,1,5,7, QtCore.Qt.AlignTop)
self.fsn_mode_widget.setLayout(fsn_mode_layout)
self.fsn_mode_scroll_view = QtGui.QScrollArea()
self.fsn_mode_scroll_view.setWidget(self.fsn_mode_widget)
self.fsn_mode_scroll_view.setWidgetResizable(True)
self.fsn_mode_scroll_view.setFixedHeight(400)
#CSV Mode Widget
self.csv_mode_widget = QtGui.QWidget()
self.input_data_set_button = ImageButton(os.path.join("essentials","csv_file.png"), 50, 50)
self.input_data_set_button.setToolTip("Click to select a data file if you want manual control.")
self.check_icons_button = QtGui.QPushButton("Check Icon Availability\nand Export Report")
csv_mode_layout = QtGui.QHBoxLayout()
csv_mode_layout.addStretch(1)
csv_mode_layout.addWidget(self.input_data_set_button,0)
csv_mode_layout.addWidget(self.check_icons_button,0)
csv_mode_layout.addStretch(1)
self.csv_mode_widget.setLayout(csv_mode_layout)
self.fsn_or_csv_stacked_widget = QtGui.QStackedWidget()
self.fsn_or_csv_stacked_widget.addWidget(self.fsn_mode_scroll_view)
self.fsn_or_csv_stacked_widget.addWidget(self.csv_mode_widget)
self.validate_button = ImageButton(os.path.join("essentials","validate.png"),50,50)
layout = QtGui.QGridLayout()
layout.addWidget(self.page_selector,0,0,1,2, QtCore.Qt.AlignHCenter)
layout.addWidget(self.fsn_or_csv_stacked_widget,1,0,1,2)
layout.addWidget(self.validate_button,3,1)
self.group_box.setLayout(layout)
final_layout = QtGui.QHBoxLayout()
final_layout.addWidget(self.group_box)
self.setLayout(final_layout)
class DataSelector(QtGui.QWidget):
def __init__(self, repo_path):
super(DataSelector,self).__init__()
self.repo_path = repo_path
self.data_from_fk = None
self.createUI()
self.fk_retriever = FKRetriever(self.repo_path)
self.mapEvents()
self.data_is_ready = False
self.data = None
self.validate_button.setEnabled(False)
def createUI(self):
self.group_box = QtGui.QGroupBox("Data Selector")
self.page_selector = IconListBox()
page_control_list = [
{
"Name": "From Flipkart Using FSNs",
"Icon": os.path.join("essentials","download.png")
},
{
"Name": "From CSV Data File",
"Icon": os.path.join("essentials","csv_file.png")
}
]
self.page_selector.addElements(page_control_list)
self.page_selector.setFixedSize(302,110)
#FSN Mode Widget
self.fsn_mode_widget = QtGui.QGroupBox("Data By FSN")
self.fsn_text_edit = FSNTextEdit()
self.fsn_text_edit.setFixedSize(450,400)
self.category_label = QtGui.QLabel("Category:")
self.category_combo_box = QtGui.QComboBox()
self.category_combo_box.addItems(getCategoryFolderNames()) #Later, add this data from OINK's server.
self.category_combo_box.setToolTip("Select the default category for the given FSNs.\nNote that mixing various types of FSNs isn't recommended.\nThe icons won't load.")
self.attributes_list_box = QtGui.QListWidget()
self.attributes_list_box.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection)
self.attributes_list_box.setToolTip("Displays all product attributes, obtained from the FK server.")
self.primary_attributes_list_box = QtGui.QListWidget()
self.primary_attributes_list_box.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection)
self.primary_attributes_list_box.setToolTip("Displays primary product attributes that you have selected.")
self.secondary_attributes_list_box = QtGui.QListWidget()
self.secondary_attributes_list_box.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection)
self.secondary_attributes_list_box.setToolTip("Displays secondary product attributes that you have selected.")
self.push_to_primary_button = QtGui.QPushButton("Add to\nPrimary List")
self.push_to_primary_button.setToolTip("Click to move the chosen attribute into the list of primary attributes.")
self.remove_from_primary_button = QtGui.QPushButton("Remove from\nPrimary List")
self.remove_from_primary_button.setToolTip("Click to move the chosen attribute out of the list of primary attributes.")
self.push_to_secondary_button = QtGui.QPushButton("Add to\nSecondary List")
self.push_to_secondary_button.setToolTip("Click to move the chosen attribute into the list of secondary attributes.")
self.remove_from_secondary_button = QtGui.QPushButton("Remove from\nSecondary List")
self.remove_from_secondary_button.setToolTip("Click to move the chosen attribute out of the list of secondary attributes.")
#downloader
self.fetch_images_attributes_button = QtGui.QPushButton("Download")
self.fetch_images_attributes_button.setToolTip("This will check if parent images are available for all the FSNs and download them if necessary from the FK site. It will also load the spec table.")
self.export_scraped_data_button = QtGui.QPushButton("Export Data")
self.fetching_progress = ProgressBar()
self.fetching_progress.setRange(0,100)
self.fetching_progress.setValue(0)
self.fetching_activity = QtGui.QLabel("All that is gold does not glitter!")
self.fetching_activity.setStyleSheet("QLabel{font: 10px black; border: 1px solid black;}")
self.fetching_activity.setToolTip("This indicates the current downloader's activity, or some random quote that Vinay thinks is funny.")
self.completed_fsns_count_label = QtGui.QLabel("Completed:")
self.completed_fsns_count_spinbox = QtGui.QSpinBox()
self.completed_fsns_count_spinbox.setEnabled(False)
self.eta_label = QtGui.QLabel("ETA:")
self.eta_datetimeedit = QtGui.QDateTimeEdit()
self.eta_datetimeedit.setEnabled(False)
self.eta_datetimeedit.setMinimumDateTime(QtCore.QDateTime(datetime.datetime.now()))
self.activity_log_textedit = QtGui.QTextEdit()
self.activity_log_textedit.setReadOnly(True)
self.pending_fsns_count_label = QtGui.QLabel("Pending:")
self.pending_fsns_count_spinbox = QtGui.QSpinBox()
self.pending_fsns_count_spinbox.setEnabled(False)
self.failed_fsns_label = QtGui.QLabel("Failed:")
self.failed_fsns_count_spinbox = QtGui.QSpinBox()
self.failed_fsns_count_spinbox.setEnabled(False)
self.completed_fsns_count_spinbox.setRange(0,99999999)
self.pending_fsns_count_spinbox.setRange(0,99999999)
self.failed_fsns_count_spinbox.setRange(0,99999999)
self.pending_fsns_list_text_edit = QtGui.QTextEdit()
self.pending_fsns_list_text_edit.setReadOnly(True)
self.completed_fsns_list_text_edit = QtGui.QTextEdit()
self.completed_fsns_list_text_edit.setReadOnly(True)
self.failed_fsns_list_text_edit = QtGui.QTextEdit()
self.failed_fsns_list_text_edit.setReadOnly(True)
buttons_and_progress_bar = QtGui.QHBoxLayout()
buttons_and_progress_bar.addWidget(self.fetch_images_attributes_button, 2)
buttons_and_progress_bar.addWidget(self.export_scraped_data_button, 1)
buttons_and_progress_bar.addWidget(self.fetching_progress, 4)
completed_tracking = QtGui.QVBoxLayout()
completed_tracking.addWidget(self.completed_fsns_count_label)
completed_tracking.addWidget(self.completed_fsns_count_spinbox)
completed_tracking.addWidget(self.completed_fsns_list_text_edit)
eta_layout = QtGui.QHBoxLayout()
eta_layout.addWidget(self.eta_label)
eta_layout.addWidget(self.eta_datetimeedit)
pending_tracking = QtGui.QVBoxLayout()
pending_tracking.addWidget(self.pending_fsns_count_label)
pending_tracking.addWidget(self.pending_fsns_count_spinbox)
pending_tracking.addWidget(self.pending_fsns_list_text_edit)
pending_tracking.addLayout(eta_layout)
failed_tracking = QtGui.QVBoxLayout()
failed_tracking.addWidget(self.failed_fsns_label)
failed_tracking.addWidget(self.failed_fsns_count_spinbox)
failed_tracking.addWidget(self.failed_fsns_list_text_edit)
fsns_tracking = QtGui.QHBoxLayout()
fsns_tracking.addLayout(completed_tracking)
fsns_tracking.addLayout(pending_tracking)
fsns_tracking.addLayout(failed_tracking)
downloader_layout = QtGui.QVBoxLayout()
downloader_layout.addLayout(buttons_and_progress_bar)
downloader_layout.addLayout(fsns_tracking)
downloader_layout.addWidget(self.fetching_activity)
downloader = QtGui.QWidget()
downloader.setLayout(downloader_layout)
downloader_tabs = QtGui.QTabWidget()
downloader_tabs.addTab(downloader, "Downloader")
downloader_tabs.addTab(self.activity_log_textedit, "Log")
self.fsn_mode_data_options = QtGui.QGroupBox("Data Options")
fsn_mode_data_options_layout = QtGui.QGridLayout()
fsn_mode_data_options_layout.addWidget(self.category_label,0,0,1,2, QtCore.Qt.AlignVCenter)
fsn_mode_data_options_layout.addWidget(self.category_combo_box,0,2,1,2,QtCore.Qt.AlignVCenter)
fsn_mode_data_options_layout.addWidget(self.attributes_list_box,1,0,4,4, QtCore.Qt.AlignHCenter)
fsn_mode_data_options_layout.addWidget(self.primary_attributes_list_box,1,5,2,2, QtCore.Qt.AlignHCenter)
fsn_mode_data_options_layout.addWidget(self.push_to_primary_button,1,4,1,1, QtCore.Qt.AlignHCenter | QtCore.Qt.AlignVCenter)
fsn_mode_data_options_layout.addWidget(self.remove_from_primary_button,2,4,1,1, QtCore.Qt.AlignHCenter | QtCore.Qt.AlignVCenter)
fsn_mode_data_options_layout.addWidget(self.secondary_attributes_list_box,3,5,2,2, QtCore.Qt.AlignHCenter)
fsn_mode_data_options_layout.addWidget(self.push_to_secondary_button,3,4,1,1, QtCore.Qt.AlignHCenter | QtCore.Qt.AlignVCenter)
fsn_mode_data_options_layout.addWidget(self.remove_from_secondary_button,4,4,1,1, QtCore.Qt.AlignHCenter | QtCore.Qt.AlignVCenter)
self.fsn_mode_data_options.setLayout(fsn_mode_data_options_layout)
self.fsn_mode_data_options.setEnabled(False)
fsn_mode_layout = QtGui.QGridLayout()
fsn_mode_layout.addWidget(self.fsn_text_edit,0,0,7,1)
downloader_tabs
fsn_mode_layout.addWidget(downloader_tabs,0,1,2,5, QtCore.Qt.AlignBottom)
#fsn_mode_layout.addWidget(self.fetch_images_attributes_button,0,1,1,2, QtCore.Qt.AlignBottom)
#fsn_mode_layout.addWidget(self.fetching_progress,0,3,1,5, QtCore.Qt.AlignBottom)
#fsn_mode_layout.addWidget(self.fetching_activity,1,3,1,5, QtCore.Qt.AlignTop)
#fsn_mode_layout.addWidget(self.export_scraped_data_button,1,1,1,2, QtCore.Qt.AlignTop)
fsn_mode_layout.addWidget(self.fsn_mode_data_options,2,1,5,7, QtCore.Qt.AlignTop)
self.fsn_mode_widget.setLayout(fsn_mode_layout)
self.fsn_mode_scroll_view = QtGui.QScrollArea()
self.fsn_mode_scroll_view.setWidget(self.fsn_mode_widget)
self.fsn_mode_scroll_view.setWidgetResizable(True)
self.fsn_mode_scroll_view.setFixedHeight(400)
#CSV Mode Widget
self.csv_mode_widget = QtGui.QWidget()
self.input_data_set_button = ImageButton(os.path.join("essentials","csv_file.png"), 50, 50)
self.input_data_set_button.setToolTip("Click to select a data file if you want manual control.")
self.check_icons_button = QtGui.QPushButton("Check Icon Availability\nand Export Report")
csv_mode_layout = QtGui.QHBoxLayout()
csv_mode_layout.addStretch(1)
csv_mode_layout.addWidget(self.input_data_set_button,0)
csv_mode_layout.addWidget(self.check_icons_button,0)
csv_mode_layout.addStretch(1)
self.csv_mode_widget.setLayout(csv_mode_layout)
self.fsn_or_csv_stacked_widget = QtGui.QStackedWidget()
self.fsn_or_csv_stacked_widget.addWidget(self.fsn_mode_scroll_view)
self.fsn_or_csv_stacked_widget.addWidget(self.csv_mode_widget)
self.validate_button = ImageButton(os.path.join("essentials","validate.png"),50,50)
layout = QtGui.QGridLayout()
layout.addWidget(self.page_selector,0,0,1,2, QtCore.Qt.AlignHCenter)
layout.addWidget(self.fsn_or_csv_stacked_widget,1,0,1,2)
layout.addWidget(self.validate_button,3,1)
self.group_box.setLayout(layout)
final_layout = QtGui.QHBoxLayout()
final_layout.addWidget(self.group_box)
self.setLayout(final_layout)
def mapEvents(self):
self.page_selector.currentItemChanged.connect(self.changePage)
self.input_data_set_button.clicked.connect(self.loadDataFromFile)
self.fetch_images_attributes_button.clicked.connect(self.downloadFromFK)
self.fk_retriever.sendData.connect(self.prepareDataRetrievedFromFK)
self.fk_retriever.sendException.connect(self.postException)
self.push_to_primary_button.clicked.connect(self.pushAttrToPrimary)
self.remove_from_primary_button.clicked.connect(self.removeFromPrimary)
self.push_to_secondary_button.clicked.connect(self.pushAttrToSecondary)
self.remove_from_secondary_button.clicked.connect(self.removeFromSecondary)
self.category_combo_box.currentIndexChanged.connect(self.changeCategory)
self.export_scraped_data_button.clicked.connect(self.exportData)
self.check_icons_button.clicked.connect(self.checkIconsAvailability)
def checkIconsAvailability(self):
import pandas as pd
import xlsxwriter
import Katana
data = self.getData()
icons = {}
repo_path = str(QtGui.QFileDialog.getExistingDirectory(self, "Select the repository folder.", os.getcwd(),
QtGui.QFileDialog.ShowDirsOnly | QtGui.QFileDialog.DontResolveSymlinks))
if not repo_path:
repo_path = os.path.join(os.getcwd(), "Images","Repository")
for fsn_row in self.getData():
category = fsn_row["Category"]
for key in fsn_row.keys():
if "Attribute" in key:
attribute = fsn_row[key].strip()
if attribute.strip() != "":
icon_status, folders = Katana.checkIcon(attribute, category, repository_path=repo_path)
if attribute not in icons.keys():
icons[attribute] = {
"Category": category,
"Icon in Folder(s)": folders
}
else:
if category not in icons[attribute]["Category"]:
icons[attribute]["Category"] = icons[attribute]["Category"] + ", " +category
icons[attribute]["Icon in Folder(s)"] = [folder for folder in list(set(icons[attribute]["Icon in Folder(s)"] + folders)) if len(folder)>0]
icons_data_frame = pd.DataFrame.from_dict(icons)
icons_data_frame = icons_data_frame.apply(self.getMeaningfulPathText,axis=0)
file_path = os.path.join(os.getcwd(),"cache","Icon_Search_Results_%s.csv"%datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
#file_handler = pd.ExcelWriter(file_path,engine="xlsxwriter")
icons_data_frame.T.to_csv(file_path)
os.startfile(file_path,"open")
#icons_data_frame.to_excel(file_handler, "Sheet1")
#file_handler.save()
# print "Saved file to %s!"%file_path
def getMeaningfulPathText(self, text):
if type(text["Icon in Folder(s)"]) == str:
return text
elif type(text["Icon in Folder(s)"]) == list:
if len(text["Icon in Folder(s)"]) == 0:
text["Icon in Folder(s)"] = "No Icons Available."
else:
text["Icon in Folder(s)"] = ", ".join(text["Icon in Folder(s)"])
else:
print "What happened?"
print text
return text
def exportData(self):
import pandas as pd
import xlsxwriter
#Get the output location.
output_path = str(QtGui.QFileDialog.getExistingDirectory(self, "Select the output folder.", os.getcwd(),
QtGui.QFileDialog.ShowDirsOnly | QtGui.QFileDialog.DontResolveSymlinks))
#Calculate the different types of FSNs based on prefix.
if output_path:
file_path = os.path.join(output_path,"flipkart_data_%s.xlsx"%datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))
file_handler = pd.ExcelWriter(file_path,engine="xlsxwriter")
fsns_list = self.data_from_fk.keys()
prefixes = list(set([fsn[:3] for fsn in fsns_list]))
prefixes.sort()
seggregated_data_set = {}
#Split data based on FSN prefix.
for prefix in prefixes:
valid_fsns = [fsn for fsn in fsns_list if prefix==fsn[:3]]
seggregated_data_set[prefix] = {}
for fsn in valid_fsns:
seggregated_data_set[prefix][fsn] = self.data_from_fk[fsn]
#Create dataframes for each data set.
prefix_data_set = pd.DataFrame.from_dict(seggregated_data_set[prefix])
#Save the dataframe in an excel file with the entire dataset in one sheet,
#and individual sheets containing prefix-wise data, stored in the output folder.
prefix_data_set.T.to_excel(file_handler, prefix)
file_handler.save()
pd.DataFrame.from_dict(self.data_from_fk).T.to_csv(os.path.join(output_path,"flipkart_data_%s.csv"%datetime.datetime.now().strftime("%Y%m%d_%H%M%S")))
os.startfile(file_path,"open")
self.sendAlert("Success","Success exported data for %d possible verticals into %s."%(len(prefixes),os.path.basename(file_path)))
# data_frame = pd.DataFrame.from_dict(self.data_from_fk).transpose()
# data_frame.to_csv("something.csv")
#get save file name
#dump data into said file.
def postException(self, error_msg):
message = "%s @%s"%(error_msg,datetime.datetime.now().strftime("%H:%M:%S"))
self.fetching_activity.setText(message)
self.activity_log_textedit.append(message)
def pushAttrToPrimary(self):
self.pushFromTo(self.attributes_list_box,self.primary_attributes_list_box)
def removeFromPrimary(self):
self.pushFromTo(self.primary_attributes_list_box,self.attributes_list_box)
def pushAttrToSecondary(self):
self.pushFromTo(self.attributes_list_box,self.secondary_attributes_list_box)
def removeFromSecondary(self):
self.pushFromTo(self.secondary_attributes_list_box,self.attributes_list_box)
def pushFromTo(self,source_list_widget, destination_list_widget):
#identify the selected attributes
selected_attribute_items = source_list_widget.selectedItems()
#Store them in a list.
selected_attributes = [str(selected_attribute_item.text()) for selected_attribute_item in selected_attribute_items]
#Send them to the destination list.
destination_list_widget.setSortingEnabled(False)
destination_list_widget.addItems(selected_attributes)
destination_list_widget.setSortingEnabled(True)
destination_list_widget.sortItems()
#Remove them from the source list
for selected_item in selected_attribute_items:
source_list_widget.takeItem(source_list_widget.row(selected_item))
self.makeDataFile()
def makeDataFile(self):
"""Creates a list of dictionaries for the FSNs, using the retrieved data."""
#Extract the primary and secondary attributes.
primary_attributes = [str(self.primary_attributes_list_box.item(list_index).text()) for list_index in range(self.primary_attributes_list_box.count())]
secondary_attributes = [str(self.secondary_attributes_list_box.item(list_index).text()) for list_index in range(self.secondary_attributes_list_box.count())]
#algorithm
#Build a list of dictionaries with the following structure:
output_data_format = [
{
"FSN": None,
"Category": None,
"Primary USP-1 Attribute": None,
"Primary USP-1 Description Text":None,
"Primary USP-2 Attribute": None,
"Primary USP-2 Description Text":None,
"Secondary USP-1 Attribute": None,
"Secondary USP-1 Description Text":None,
"Secondary USP-2 Attribute": None,
"Secondary USP-2 Description Text":None,
}
]
#In doing this, check if FSNs have far too many attributes selected, or if they have none at all.
#To check, see if the attribute is in the fsn_data_set that FKRetriever passes.
#End algorithm
output_data = []
category = str(self.category_combo_box.currentText())
if self.data_from_fk is None:
self.sendAlert("Cowabunga!","Something seems to be wrong. This situation shouldn't ever happen. If there are attributes populated in the list widgets, then this shouldn't ever happen. This indicates that Leonardo failed to retrieve information from the Flipkart website or API. But if the attributes list widgets are populated, then this is ridiculously impossible.")
else:
never = False
if (len(primary_attributes)>0) and (len(secondary_attributes)>0):
#loop through each fsn key.
invalid_fsns = []
total_fsns = len(self.data_from_fk.keys())
for fsn in self.data_from_fk:
fsn_data = self.data_from_fk[fsn]
fsn_attributes_mapping = {
"FSN": fsn,
"Category": category
}
primary_attribute_counter = 0
for primary_attribute in primary_attributes:
if primary_attribute in fsn_data.keys():
primary_attribute_counter += 1
attr_key = "Primary USP-%d Attribute"%primary_attribute_counter
descr_key = "Primary USP-%d Description Text"%primary_attribute_counter
#Check if the icon exists. If it doesn't compile a list of icons required.
#icon_available = checkIcon(attr_key,descr_key)
fsn_attributes_mapping.update({attr_key:primary_attribute,descr_key: fsn_data[primary_attribute]})
secondary_attribute_counter = 0
for secondary_attribute in secondary_attributes:
if secondary_attribute in fsn_data.keys():
secondary_attribute_counter += 1
attr_key = "Secondary USP-%d Attribute"%secondary_attribute_counter
descr_key = "Secondary USP-%d Description Text"%secondary_attribute_counter
#Check if the icon exists. If it doesn't compile a list of icons required.
#icon_available = checkIcon(attr_key,descr_key)
fsn_attributes_mapping.update({attr_key:secondary_attribute,descr_key: fsn_data[secondary_attribute]})
if (primary_attribute_counter == 0) or (secondary_attribute_counter == 0):
invalid_fsns.append(fsn)
output_data.append(fsn_attributes_mapping)
if len(invalid_fsns)>0:
message = "There are %d fsns without enough primary or secondary attributes. Trying this process for FSNs of mixed category\sub-category isn't recommended." %len(invalid_fsns)
self.sendAlert("Uh-oh!",message)
self.validate_button.setStyleSheet("background-color: #B22222")
self.validate_button.setEnabled(False)
else:
self.data = output_data
self.validate_button.setEnabled(True)
self.validate_button.setStyleSheet("QPushButton{background-color: #458B00} QPushButton:hover{background-color: #78AB46};")
elif never:
#(len(primary_attributes) == 0) or (len(secondary_attributes) == 0):
#This could be a problem in runtime. Disabling for now.
self.sendAlert("Cowabunga!","Please promote some attributes to primary and secondary positions. If you don't want to use secondary attributes, just add one anyway, and select equal relative icon sizes later.")
def changeCategory(self):
self.makeDataFile()
def sendAlert(self, title, message):
QtGui.QMessageBox.about(self, title, message)
def downloadFromFK(self):
"""Triggers FKRetriever."""
fsns = self.fsn_text_edit.getFSNs()
self.fetch_images_attributes_button.setEnabled(False)
if len(fsns) >=1:
self.fetching_activity.setText("Preparing to download images and specifications off the Flipkart website!")
self.fk_retriever.fsn_list = fsns
self.fk_retriever.allow_run = True
else:
print "No FSNS to process."
def prepareDataRetrievedFromFK(self, status, data_set, progress_value, fsn_lists, completion_status, eta):
"""Gets data from FK from the thread's signal and prepares it."""
self.fetching_progress.setValue(progress_value)
eta_string = eta.strftime("%a (%d-%b), %H:%M:%S")
self.putAttributes(data_set)
self.data_from_fk = data_set
now_string = datetime.datetime.now().strftime("(%d-%b), %H:%M:%S")
completed_fsns = fsn_lists[0]
pending_fsns = fsn_lists[1]
failed_fsns = fsn_lists[2]
self.completed_fsns_count_spinbox.setValue(len(completed_fsns))
self.pending_fsns_count_spinbox.setValue(len(pending_fsns))
self.failed_fsns_count_spinbox.setValue(len(failed_fsns))
self.completed_fsns_list_text_edit.setText("\n".join(completed_fsns))
self.pending_fsns_list_text_edit.setText("\n".join(pending_fsns))
self.failed_fsns_list_text_edit.setText("\n".join(failed_fsns))
self.eta_datetimeedit.setDateTime(QtCore.QDateTime(eta))
if completion_status:
self.fetching_progress.setFormat("%d%% (Completed at %s)"%(progress_value, now_string))
message = "Completed at %s."%now_string
self.fsn_mode_data_options.setEnabled(True)
self.fetch_images_attributes_button.setEnabled(True)
self.sendAlert("Cowabunga!","Completed fetching data and images for the given list.")
else:
self.fetching_progress.setFormat("%d%% @(%s)"%(progress_value, now_string))
message = "%s ETA: %s"%(status, eta_string)
self.fsn_mode_data_options.setEnabled(False)
self.fetching_activity.setText(message)
self.activity_log_textedit.append(message)
def putAttributes(self, data_set):
attributes = []
for fsn in data_set:
attributes+=data_set[fsn].keys()
attributes = list(set(attributes))
self.attributes_list_box.setSortingEnabled(False)
self.attributes_list_box.clear()
self.primary_attributes_list_box.clear()
self.secondary_attributes_list_box.clear()
self.attributes_list_box.addItems(attributes)
self.attributes_list_box.setSortingEnabled(True)
self.attributes_list_box.sortItems()
def getData(self):
return self.data
def changePage(self, current, previous):
if not current:
current = previous
self.fsn_or_csv_stacked_widget.setCurrentIndex(self.page_selector.row(current))
def loadDataFromFile(self):
"""This method asks for a csv data file. Upon loading, it'll read the file,
check it and declare whether it's valid or not.
It does it based on:
1. File headers:
FSN, Brand, Category, Primary USP[1-5] Attribute; Primary USP[1-5] Description; Secondary USP[1-5] Attribute; Secondary USP[1-5] Description;
2. At least 1 row of data.
"""
#Get the file name.
data_file_name = str(QtGui.QFileDialog.getOpenFileName(self,"Open Data File",os.getcwd(),("Comma Separated Values Files (*.csv)")))
if data_file_name:
#Load the file.
data_file_handler = open(data_file_name,"r")
data_file_as_csv = csv.DictReader(data_file_handler)
file_headers = []
for row in data_file_as_csv:
file_headers = row.keys()
file_headers.sort()
required_file_headers = [
"FSN","Brand","Category",
"Primary USP-1 Attribute","Primary USP-1 Description Text",
"Primary USP-2 Attribute","Primary USP-2 Description Text",
"Primary USP-3 Attribute","Primary USP-3 Description Text",
"Primary USP-4 Attribute","Primary USP-4 Description Text",
"Primary USP-5 Attribute","Primary USP-5 Description Text",
"Secondary USP-1 Attribute","Secondary USP-1 Description Text",
"Secondary USP-2 Attribute","Secondary USP-2 Description Text",
"Secondary USP-3 Attribute","Secondary USP-3 Description Text",
"Secondary USP-4 Attribute","Secondary USP-4 Description Text",
"Secondary USP-5 Attribute","Secondary USP-5 Description Text"
]
required_file_headers.sort()
data_is_valid = True
for header in required_file_headers:
if header not in file_headers:
data_is_valid = False
self.sendAlert("Wrong Data set.", "%s column is required!"%header)
break
if data_is_valid:
data_file_handler.seek(0)
next(data_file_handler) #0 has the header, so go to row 1.
self.data = []
for row in data_file_as_csv:
if len(row["FSN"].strip()) > 0:
self.data.append(row)
if len(self.data)>0:
self.data_is_ready = True
self.sendAlert("Success", "%d FSNs have been uploaded from the provided dataset"%len(self.data))
self.validate_button.setEnabled(True)
self.validate_button.setStyleSheet("QPushButton{background-color: #458B00} QPushButton:hover{background-color: #78AB46};")
else:
self.data_is_ready = False
self.sendAlert("No FSNs in the data set", "The Data set has the right columns but it appears to not have any rows with data.")
self.validate_button.setStyleSheet("background-color: #B22222")
self.validate_button.setStyleSheet("background-color: #B22222")
else:
self.validate_button.setStyleSheet("background-color: #B22222")
self.validate_button.setEnabled(False)
data_file_handler.close()
# set up keyboard variables
isMoveLeft = False
isMoveRight = False
isMoveUp = False
isMoveDown = False
oneThird = 1/3
# set up music
kaboomSound = pygame.mixer.Sound('res/sound/kaboom.wav')
laserSound = pygame.mixer.Sound('res/sound/laser_blast.wav')
pygame.mixer.music.load('res/sound/background.mp3')
pygame.mixer.music.play(-1, 0.0)
musicPlaying = True
laserBar = ProgressBar(5, 5, 200, 5, [255,0,0])
shieldBar = ProgressBar(5, 15, 200, 5, [0,0,255])
healthBar = ProgressBar(5, 25, 200, 5, [0, 255, 0])
def checkQuit(event):
if event.type == QUIT:
pygame.quit()
sys.exit()
def quit():
pygame.quit()
sys.exit()
def addAsteroid():
global asteroidCounter
global WINDOWWIDTH
asteroidCounter += 1
