def findClosestLyrics(data, inpData, kth, x, y):
"""Applies the kth nearst algoritm using word count as distance
Parameters:
data (list): Lyric data that will be used for comparing inpData
inpData (str): Input lyric data
kth (int): how many neaighbort
x (int): Print the algorithm procces onto temrianl window x
y (int): Print the algorithm procces onto temrianl window y
Returns:
list: kth nearset values
"""
maxList = MaxList(kth, lambda it, inp: inp['dist'] < it['dist'])
count = 0
badStrings = 0
beginTime = currentMilliTime()
lastTimePrinted = 0
rcount = len(data)
for elem in data:
if type(inpData) is str and type(elem[3]) is str:
dis = wordCountDistance(re.findall(r'\w+', inpData),
re.findall(r'\w+', elem[3]))
maxList.pushAndReplace({
"genre": elem[2],
"song": elem[0],
"artist": elem[1],
"dist": dis
})
else:
badStrings += 1
count += 1
currentMili = currentMilliTime()
if count >= rcount - 5 or currentMili - lastTimePrinted > 500:
lastTimePrinted = currentMili
progressBar(
count, rcount, " Completed, Time Elapsed ~= " + str(
(currentMili - beginTime) / 1000) + " secs, " +
str(count) + " Songs compared, Bad Data:" + str(badStrings), x,
y)
return maxList.getItems()
def mcmc(Chain, runtime, mixingtime=1000):
#mixingtime = 1000
#runtime = 50000
pb = progressBar(maxValue = runtime)
#dit = pickle.load(open('%s/bestpar.p' % cwd, 'r'))
#GdotOG, KD = dit['BEST']
GdotOG, KD = bestparameters
MarkovChain = Chain
best = (GdotOG, KD)
p = probcal(GdotOG, KD)
p0 = p
n = count()
m = count()
while n.next() < mixingtime + runtime:
NKD = KD + normal(0, 1.e-3)
NGdotOG = GdotOG + normal(0, 1.e-12)
p1 = probcal(NGdotOG, NKD)
c = m.next()
if p1 > p0:
if c > mixingtime:
MarkovChain.append((NGdotOG, NKD ))
pb(c - mixingtime)
p0 = p1
KD, GdotOG = NKD, NGdotOG
if p1 > p:
best = (NGdotOG, NKD)
p = p1
else:
t = uniform(0,1,1)[0]
if t < p1/p0:
if c > mixingtime:
MarkovChain.append((NGdotOG, NKD))
pb(c - mixingtime)
p0 = p1
KD, GdotOG = NKD, NGdotOG
else:
if c > mixingtime:
MarkovChain.append((GdotOG, KD))
#print MarkovChain
print 'Best Gdot/G, Kappa_D:', best
print '%d new points generated.' %len(MarkovChain)
def mcmc(Chain, runtime, mixingtime=1000):
#mixingtime = 1000
#runtime = 50000
pb = progressBar(maxValue=runtime)
#dit = pickle.load(open('%s/bestpar.p' % cwd, 'r'))
#GdotOG, KD = dit['BEST']
GdotOG, KD = bestparameters
MarkovChain = Chain
best = (GdotOG, KD)
p = probcal(GdotOG, KD)
p0 = p
n = count()
m = count()
while n.next() < mixingtime + runtime:
NKD = KD + normal(0, 1.e-3)
NGdotOG = GdotOG + normal(0, 1.e-12)
p1 = probcal(NGdotOG, NKD)
c = m.next()
if p1 > p0:
if c > mixingtime:
MarkovChain.append((NGdotOG, NKD))
pb(c - mixingtime)
p0 = p1
KD, GdotOG = NKD, NGdotOG
if p1 > p:
best = (NGdotOG, NKD)
p = p1
else:
t = uniform(0, 1, 1)[0]
if t < p1 / p0:
if c > mixingtime:
MarkovChain.append((NGdotOG, NKD))
pb(c - mixingtime)
p0 = p1
KD, GdotOG = NKD, NGdotOG
else:
if c > mixingtime:
MarkovChain.append((GdotOG, KD))
#print MarkovChain
print 'Best Gdot/G, Kappa_D:', best
print '%d new points generated.' % len(MarkovChain)
def mcmc(Chain, runtime, mixingtime=1000):
#mixingtime = 1000
#runtime = 50000
pb = progressBar(maxValue=runtime)
cwd = os.getcwd()
tmpdir = cwd + '/' + uniquename()
if not tmpdir == None:
if os.path.exists(tmpdir):
os.chdir(tmpdir)
else:
os.mkdir(tmpdir)
os.chdir(tmpdir)
os.system('cp %s/%s %s/%s' % (cwd, parfile, tmpdir, parfile))
motifile(toafile, cwd, tmpdir)
dict = pickle.load(open('%s/bestpar.p' % cwd, 'r'))
Omega, cosi, m2 = dict['BEST']
MarkovChain = Chain
pf = PARfile(parfile)
pf.thawall()
pf.write()
pf.matrix(toafile)
pf.freezeall()
pf.write()
p0 = probcal(Omega, cosi, m2, pf)
p = p0
best = (Omega, cosi, m2)
#dict = {'BEST':best}
#pickle.dump(dict, open('bestpar.p', 'w'))
n = count()
m = count()
while n.next() < mixingtime + runtime:
npf = pf.randomnew()
Ncosi = cosi + uniform(-0.05, 0.05)
#Nm2 = m2 + normal(0, 0.02)
NOmega = (Omega + normal(0, 5))
#Nsini = float(str(npf.SINI[0]))
#Ncosi = -1. * sqrt(1 - Nsini**2)
Nm2 = float(str(npf.M2[0]))
#print Ncosi, cosi, Ncosi - cosi, Nm2 - m2
p1 = probcal(NOmega, Ncosi, Nm2, npf)
c = m.next()
if p1 > p0:
if c > mixingtime:
MarkovChain.append((NOmega, Ncosi, Nm2))
pb(c - mixingtime)
pf = npf
p0 = p1
cosi, m2, Omega = Ncosi, Nm2, NOmega
if p1 > p:
best = (NOmega, Ncosi, Nm2)
p = p1
dict['BEST'] = best
pickle.dump(dict, open('%s/bestpar.p' % cwd, 'w'))
else:
t = uniform(0, 1, 1)[0]
if t < p1 / p0:
if c > mixingtime:
MarkovChain.append((NOmega, Ncosi, Nm2))
pb(c - mixingtime)
pf = npf
p0 = p1
cosi, m2, Omega = Ncosi, Nm2, NOmega
else:
if c > mixingtime:
MarkovChain.append((Omega, cosi, m2))
#print MarkovChain
print best
print '%d new points generated.' % len(MarkovChain)
#OldChain = pickle.load(open('MChain.p','r'))['Chain']
#dict['Chain'] = OldChain + MarkovChain
#dict['Chain'] = MarkovChain
#os.rmdir(tmpdir)
os.chdir(cwd)
def readFromOracle(self, iovStartUnix, iovEndUnix, folderChannelDict,
folderVariableDict):
"""
"""
#=== get logging status
isDebug = (logging.getLogger().getEffectiveLevel() <= logging.DEBUG)
#=== verify and translate times, maximum is current time
now = int(time.time())
iovStart = self.getOracleTimeString(iovStartUnix)
iovEnd = self.getOracleTimeString(now)
if iovEndUnix > iovStartUnix and iovEndUnix > 0:
iovEnd = self.getOracleTimeString(iovEndUnix)
print "IOV start: ", time.ctime(
iovStartUnix
), " (in local time)\t ORACLE time string: ", iovStart, " (in UTC)"
print "IOV end : ", time.ctime(
iovEndUnix
), " (in local time)\t ORACLE time string: ", iovEnd, " (in UTC)"
#=== initialize return list
folderVarSetList = []
#=== get all tables spanned by query
tableRange = self.getEventHistoryTables(iovStart, iovEnd)
tables = tableRange.keys()
tables.sort()
print "===> Going to access the following oracle table(s):"
for table in tables:
print " * ", table, " ---> validity range: ", tableRange[table]
for folder, channels in folderChannelDict.iteritems():
logging.debug("treating folder: %s" % folder)
varsInFolder = folderVariableDict[folder]
logging.debug("---> variables in this folder: %s" % varsInFolder)
for channel in channels:
drawer = self.info.get_drawer(folder, channel)
logging.debug("channel: %s" % channel)
nCrap = 0
for var in varsInFolder:
varType = self.info.get_variable_type(var)
logging.debug("variable: %s" % var)
folderVarSet = FolderVarSet([var], self.putDuplicates)
bar = progressBar(iovStartUnix, iovEndUnix, 78, drawer,
var)
for table in tables:
tableRangeStart, tableRangeEnd = tableRange[table]
logging.debug("Processing table: %s" % table)
oracleString = self.getOracleString(
folder, drawer, var, table, tableRangeStart,
tableRangeEnd)
logging.debug("Oralce string: %s" % oracleString)
stmt = self.db.Statement(oracleString)
if stmt.Process():
stmt.StoreResult()
#=== read all values
value = 0
while stmt.NextResultRow():
if varType == self.info.type_int:
value = stmt.GetInt(1)
elif varType == self.info.type_float:
value = stmt.GetDouble(1)
#=== catch crap in oracle
if value < -10000:
nCrap += 1
continue
#=== extract time stamp
Y = stmt.GetYear(0)
M = stmt.GetMonth(0)
D = stmt.GetDay(0)
H = stmt.GetHour(0)
Min = stmt.GetMinute(0)
Sec = stmt.GetSecond(0)
tuple = (Y, M, D, H, Min, Sec, 0, 0, 0)
#=== time.mktime interprets tuple as local standard (==winter) time
#=== example: Oracle time stamp is 15:00
#=== mktime makes seconds for 15:00 local winter time, i.e. 14:00 UTC
#=== --> need to add 3600 seconds (time.timezone == -3600)
unixTime = time.mktime(tuple) - time.timezone
iovSince = int(unixTime * self.unix2cool)
folderVarSet.setVariable(var, value)
folderVarSet.registerInIOVStore(
drawer, iovSince)
if not isDebug: bar.update(unixTime)
logging.debug("%s %s: %s (%s)\t%f" %
(drawer, var, iovSince,
time.ctime(unixTime), value))
if not isDebug: bar.done()
folderVarSetList.append(folderVarSet)
if nCrap > 0:
logging.warning(
"Found %i \"crap\" entries for %s, ignored those!" %
(nCrap, var))
return folderVarSetList
def readFromCool(self, iovStartUnix, iovEndUnix, folderChannelDict,
folderVariableDict):
"""
"""
self.reconnect()
#=== verify and translate times, maximum is current time
#=== need iovStart/End as cool::ValidityKey
if not (iovStartUnix > 0):
iovStartUnix = self.R2minTime
if not (iovEndUnix > iovStartUnix and iovEndUnix > 0):
iovEndUnix = int(time.time())
iovS = cool.ValidityKey(iovStartUnix * self.unix2cool)
iovE = cool.ValidityKey(iovEndUnix * self.unix2cool)
print "IOV start: ", time.ctime(
iovS / self.unix2cool), "\t COOL time stamp: ", iovS
if iovStartUnix < self.R2minTime and iovEndUnix > self.R2minTime:
print "R1->R2 tm: ", time.ctime(
self.R2minTime
), "\t COOL time stamp: ", self.R2minTime * self.unix2cool
print "IOV end : ", time.ctime(
iovE / self.unix2cool), "\t COOL time stamp: ", iovE
#=== initialize return list
folderVarSetList = []
#=== loop over all different folders and build list of FolderVarSets
for folder, channels in folderChannelDict.iteritems():
logging.debug("treating folder: %s" % folder)
varsInFolder = folderVariableDict[folder]
logging.debug("---> variables in this folder: %s" % varsInFolder)
folderVarSet = FolderVarSet(varsInFolder, self.putDuplicates)
ar = [iovS, iovE]
#=== loop over different databases
while self.check_db(iovStartUnix, iovEndUnix, ar):
iovStart = ar[0]
iovEnd = ar[1]
coolFolder = self.dbV[self.dbCounter].getFolder(folder)
coolFolder.setPrefetchAll(False) #=== save some RAM
for channel in channels:
drawer = self.info.get_drawer(folder, channel)
logging.debug("---> treating channel %s, drawer %s" %
(channel, drawer))
channelSelection = cool.ChannelSelection(channel)
objs = coolFolder.browseObjects(iovStart, iovEnd,
channelSelection)
bar = progressBar(iovStart, iovEnd, 78, drawer)
while objs.goToNext():
obj = objs.currentRef()
iovSince = max(iovStart, obj.since())
for var in varsInFolder:
#=== pre- and append stuff for AI variables
if '/HVSET' in folder:
coolVarName = var.split(".")[1]
elif '/AI' in folder:
coolVarName = "AI_" + var + "_VALUE"
else:
coolVarName = var.upper()
sval = obj.payloadValue(coolVarName)
if sval == "NULL": sval = "0"
val = float(sval)
logging.debug(
"-------------> %s (%s) %s : %f" %
(iovSince, time.ctime(
iovSince / self.unix2cool), var, val))
folderVarSet.setVariable(var, val)
folderVarSet.registerInIOVStore(drawer, iovSince)
bar.update(iovSince)
bar.done()
#=== release server resources
objs.close()
folderVarSetList.append(folderVarSet)
return folderVarSetList
def getDCSTree(self,
iovStartUnix,
iovEndUnix,
drawers,
variables,
lastOnly=False,
firstAndLast=False):
"""
Returns a TTree containing the specified variables for the specified drawers between
iovStart and iovEnd. The variables in the output tree are called <drawer.variable>.
Input: - iovStartUnix: start time stamp in unix time
- iovEndUnix : end time stamp in unix time
- drawers : list of drawers, i.e. ['LBC13','LBC15']
- variables : list of variables, i.e. ['AI_5VMB_OUTPUT_V_VALUE','AI_15VMB_OUTPUT_I_VALUE']
Output: TTree with the requested variables. A new entry is created in the tree if any
of the variables changes its value. A weight called \"weight\" is calculated for each
entry in the tree proportional to the lenght of the IOV.
"""
#=== catch duplications
drawers = list(set(drawers))
if not len(variables):
variables = self.info.get_all_variables()
variables = list(set(variables))
logging.debug("Will extract the following variables:")
for var in variables:
logging.debug("\t- %s" % var)
#=== build a list of folders and
#=== 1) associated channel numbers and
#=== 2) associated variables in the folder
#=== that need to be accessed
folderChannelDict = {}
folderVariableDict = {}
for drawer in drawers:
for variable in variables:
folder, channel = self.info.get_folder_and_channel(
variable, drawer)
if folder not in folderChannelDict:
folderChannelDict[folder] = [channel]
else:
if channel not in folderChannelDict[folder]:
folderChannelDict[folder].append(channel)
if folder not in folderVariableDict:
folderVariableDict[folder] = [variable]
else:
if variable not in folderVariableDict[folder]:
folderVariableDict[folder].append(variable)
#===============================================================
#=== Extract data from DB
#===============================================================
folderVarSetList = []
if self.dbSource == "COOL":
folderVarSetList = self.readFromCool(iovStartUnix, iovEndUnix,
folderChannelDict,
folderVariableDict)
elif self.dbSource == "ORACLE":
folderVarSetList = self.readFromOracle(iovStartUnix, iovEndUnix,
folderChannelDict,
folderVariableDict)
elif self.dbSource == "TESTBEAM":
folderVarSetList = self.readFromOracle(iovStartUnix, iovEndUnix,
folderChannelDict,
folderVariableDict)
#=== extract list of unique iovSince
uniqueIOVs = []
uniqueVariables = []
for folderVarSet in folderVarSetList:
uniqueIOVs.extend(folderVarSet.getUniqueIOVs())
uniqueVariables.extend(folderVarSet.getUniqueVariables())
uniqueIOVs = sorted(list(set(uniqueIOVs)))
print "===> Found Number of unique IOVs: ", len(uniqueIOVs)
print "===> Will store the following variables:", uniqueVariables
#=== calculate normalized iov weights
uniqueIOVs.append(iovEndUnix * self.unix2cool)
iovWeight = {}
sumDiff = 0.
for i in xrange(len(uniqueIOVs) - 1):
iov = uniqueIOVs[i]
diff = float(uniqueIOVs[i + 1] - iov)
iovWeight[iov] = diff
sumDiff = sumDiff + diff
uniqueIOVs.pop(-1)
for iov in uniqueIOVs:
iovWeight[iov] = iovWeight[iov] / sumDiff
#=== create tree structure
treeName = "DCS_tree"
t = ROOT.TTree(treeName, treeName)
t.SetDirectory(0)
evTime = array('l', [0])
t.Branch('EvTime', evTime, 'EvTime/I')
weight = array('f', [0])
t.Branch('weight', weight, 'weight/F')
buffer = {}
for drawer, variable in uniqueVariables:
varName = drawer + "." + variable
arrayType, treeType = self.info.get_variable_type(variable)
buffer[varName] = array(arrayType, [0])
t.Branch(varName, buffer[varName], varName + treeType)
if firstAndLast:
uniqueIOVs = uniqueIOVs[:1] + uniqueIOVs[-1:]
if lastOnly:
uniqueIOVs = uniqueIOVs[-1:]
#=== fill tree ordered by iov
bar = progressBar(0, len(uniqueIOVs), 78)
iov = 0
for iovSince in uniqueIOVs:
bar.update(iov)
iov += 1
evTime[0] = int(iovSince / self.unix2cool)
weight[0] = iovWeight[iovSince]
for folderVarSet in folderVarSetList:
for drawer in folderVarSet.getDrawers():
folderVarSet.getFromIOVStore(drawer, iovSince)
for variable in folderVarSet.getVariables():
varName = drawer + "." + variable
if variable == "FORDAQ_MBHV":
buffer[varName][0] = int(
folderVarSet.getVariable(variable))
else:
buffer[varName][0] = folderVarSet.getVariable(
variable)
t.Fill()
bar.done()
t.ResetBranchAddresses()
return t
def getSyncDCSTree(self,
h2000,
drawer,
variables=[],
timeVarName="EvTime"):
"""
This function returns a tree containing the DCS variables in <variables>
for the drawer <drawer> synchronized with the input tree <h2000>. The
name of the time stamp used for synchronization is <timeVarName>.
Input: - h2000 : a TTree containing at least the variable <timeVarName>
- drawer : The drawer of interst in string from, i.e. \"LBC02\"
- variables : A python list containing all the variable names of
interest. By default all variables are attached.
- timeVarName: name of the time stamp in h2000 tree
"""
#=== avoid variable duplication
if not len(variables):
variables = self.info.get_all_variables()
variables = list(set(variables))
#=== sort variables by directories
folders = self.info.get_variables_by_folder(variables, drawer)
#=== prepare input tree for fast reading
evTime = array('i', [0])
h2000.SetBranchStatus("*", 0)
h2000.SetBranchStatus(timeVarName, 1)
h2000.SetBranchAddress(timeVarName, evTime)
nEntries = h2000.GetEntries()
#=== create the friend
friendName = drawer
t = ROOT.TTree(friendName, friendName)
t.SetDirectory(0)
buffer = []
nVars = len(variables)
previousVal = {}
for iVar in xrange(nVars):
var = variables[iVar]
arrayType, treeType = self.info.get_variable_type(var)
previousVal[var] = array(arrayType, [0])
buffer.append(array(arrayType, [0]))
t.Branch(var, buffer[iVar], var + treeType)
#=== loop over input tree h2000
timeBeg = -1
timeEnd = -1
bar = progressBar(0, nEntries, 78)
for jentry in xrange(nEntries):
#=== load current entry
ientry = h2000.LoadTree(jentry)
if ientry < 0: break
nb = h2000.GetEntry(jentry)
if nb <= 0: continue
bar.update(jentry)
#=== access cool only if IOV changed
if timeEnd == -1: timeBeg = evTime[0]
if evTime[0] != timeEnd:
timeEnd = evTime[0]
valKey = cool.ValidityKey(evTime[0] * self.unix2cool)
for folder, varlist in folders.iteritems():
coolFolder = self.db.getFolder(folder)
channel = self.info.get_channel(folder, drawer)
obj = coolFolder.findObject(valKey, channel)
#payload = obj.payload()
for var in varlist:
if '/HVSET' in folder:
coolVarName = var.split(".")[1]
elif '/AI' in folder:
coolVarName = "AI_" + var + "_VALUE"
else:
coolVarName = var.upper()
#previousVal[var][0] = payload[coolVarName]
sval = obj.payloadValue(coolVarName)
if sval == "NULL": sval = "0"
previousVal[var][0] = float(sval)
#=== fill the buffers with current value
for iVar in xrange(len(variables)):
var = variables[iVar]
buffer[iVar][0] = previousVal[var][0]
#=== fill friend tree
t.Fill()
#=== reset trees
h2000.SetBranchStatus("*", 1)
t.ResetBranchAddresses()
bar.done()
#=== print out stats
print "IOV start: ", time.ctime(timeBeg), "\t COOL time stamp: ", (
timeBeg * self.unix2cool)
print "IOV end : ", time.ctime(timeEnd), "\t COOL time stamp: ", (
timeEnd * self.unix2cool)
return t
def showMap():
global hydration
# We use concatenation
print("----------------------------")
print("You are in the " + rooms[currentRoom]["name"], "\n")
print("Your current Inventory: " + str(inventory), "\n")
# print("Your current hydration (%): " + str(hydration),"\n")
global prog
prog = progressBar(maxValue=100)
prog.updatePercentage(hydration)
print("hydration:")
prog.draw()
print("----------------------------")
if rooms[currentRoom]["name"] == "Hall":
print("""
_____________________________
! | !
! Hall | Kitchen !
!(YOU ARE | !
! HERE) | !
!---------------------------!
! | !
! Bathroom | Bedroom !
! | !
-----------------------------
""")
elif rooms[currentRoom]["name"] == "Kitchen":
print("""
_____________________________
! | !
! Hall | Kitchen !
! | (YOU ARE !
! | HERE) !
! | !
!---------------------------!
! | !
! Bathroom | Bedroom !
! | !
-----------------------------
""")
elif rooms[currentRoom]["name"] == "Bedroom":
print("""
_____________________________
! | !
! Hall | Kitchen !
! | !
! | !
! | !
!---------------------------!
! | !
! Bathroom | Bedroom !
! | (YOU ARE !
! | HERE) !
! | !
-----------------------------
""")
elif rooms[currentRoom]["name"] == "Bathroom":
print("""
_____________________________
! | !
! Hall | Kitchen !
! | !
! | !
!---------------------------!
! | !
! Bathroom | !
! (YOU ARE | !
! HERE) | Bedroom !
! | !
-----------------------------
""")
else:
print("Something unexpected occured. Stopping application & threads")
hydrationOverTimeThread.exit()
sys.exit()
print("\n\n\n\n\n\n\n\n\n\n")
if "item" in rooms[currentRoom]:
print("You have found a " + rooms[currentRoom]["item"])
print("\n")
global hydration
hydrationUpdate("-", 25)
if rooms[currentRoom]["item"] == "Water Bottle":
hydrationUpdate("+", 50)
def mcmc(Chain, runtime, MarkovChain, mixingtime=1000, stepsize=1, seed=0 ):
pb = progressBar(maxValue = runtime + mixingtime)
cwd=os.getcwd()
tmpdir = cwd+'/.'+uniquename()
if not tmpdir == None:
if os.path.exists(tmpdir):
os.chdir(tmpdir)
else:
os.mkdir(tmpdir)
os.chdir(tmpdir)
os.system('cp %s/%s %s/%s' % (cwd, parfile, tmpdir, parfile))
os.system('cp %s/%s %s/%s' % (cwd, pulsefile, tmpdir, pulsefile))
motifile(toafile, cwd, tmpdir)
touchparfile(parfile, NITS=1)
pf = PARfile(parfile)
#chisq, dof = tempofit(parfile, toafile = toafile, pulsefile = pulsefile)
pf.matrix(toafile)
pf.freezeall()
pf.thawall('JUMP')
p0 = probcal(pf)
pmax = p0
ThisChain = []
c = 0
randomlist = uniform(0,1,stepsize)
while c <= mixingtime + runtime:
c+=1
npf = pf.randomnew(stepsize=stepsize)
#randomnew(npf, stepsize) #only use this for 1713
p1 = probcal(npf)
if c % 30 == 0:pb(c)
if c > mixingtime:
t = randomlist[c-mixingtime-1]
if t < exp(p1-p0):
Chain.Chain.append([npf.__dict__[p][0] for p in plist] + [ npf.chisq])
pf = npf
p0 = p1
if p1 > pmax:
pmax = p1
bestpar['BEST'] = [npf.__dict__[p][0] for p in plist] + [ npf.chisq]
pickle.dump(bestpar, open('%s/bestpar.p' % cwd, 'wb', 0), protocol=2)
else:
Chain.Chain.append([pf.__dict__[p][0] for p in plist] + [ npf.chisq])
if c % (100+(seed%100)) == 0:
MarkovChain.extend(Chain.Chain)
ThisChain.extend(Chain.Chain)
Chain.Chain = [] #empty the list
#try:
TC = np.array(ThisChain)
dit = {'Chain':TC}
pid = str(os.getpid())
try:
os.remove(cwd+'/MChain.p'+pid)
except:pass
f = open(cwd+'/MChain.p'+pid, 'wb', 0)
pickle.dump(dit, f, protocol=2)
f.flush()
f.close()
del dit
del TC
MarkovChain.extend(Chain.Chain)
os.chdir(cwd)
]))
U.append(
TheMaker(simulator, DNA, [
SCV, Refinery, Supply_Depot, Barracks, Factory,
Factory_Tech_Lab, Siege_Tank, Siege_Tech
]))
#V.append(TheMaker(simulator, DNA, default))
#W.append(TheMaker(simulator, DNA, default))
#X.append(TheMaker(simulator, DNA, default))
#Y.append(TheMaker(simulator, DNA, default))
#Z.append(TheMaker(simulator, DNA, default))
Groups = [O, P, Q, R, S, T, U]
ct = count()
TotalSteps = 3000
pb = progressBar(minValue=0, maxValue=TotalSteps)
i = ct.next()
pb(0)
past = datetime.now()
while i < TotalSteps:
pb(i)
if i > 9 and i % 10 == 0:
ETR = (datetime.now() -
past).total_seconds() / 3600. / i * (TotalSteps - i)
EHR = int(ETR)
EMR = (ETR - EHR) * 60
print '\n', '; '.join([
'%d, %d' % (max(g, key=lambda x: x.fitness()).fitness(),
min(g, key=lambda x: x.fitness()).fitness())
for g in Groups
]), 'ETR:%dh:%dm' % (EHR, EMR), '\n'
def scramble_flashes(valid_storms_file = "", n_storms = 0, scramble_n = 5, max_n_at_once = 10000, savefig_title = ""):
df= pd.read_csv(valid_storms_file)
sub_df = df.sample(n = n_storms).reset_index()
######
dts_dict = {}
# START BY GETTING A LIST OF ALL TIME SEPARATIONS FOR THE TGFS IN THE SUB_DF
#----------------------------------------------------------------------------------------------
count = 0
groups = sub_df.groupby('TGF_ID')
for index, (name, group) in enumerate(groups):
for label in group['Cluster_ID'].values: # Go get the list of dts for each of the storms!
progressBar("Getting {} TGFS".format(n_storms), value = count, endvalue=n_storms-1)
x = clusterer.read_clusters(file='/mnt/driveA/ENTLN_Processing/Merged/merged_clustering/'+name+'.txt', specific_clusters=[label]) #Individual Storms
x.get_general_flahses()
dts = x.clust_dts # time difference for all flashes in the storm
dts_dict[count] = dts
count+=1
#----------------------------------------------------------------------------------------------
######
# NOW WE NEED TO TAKE SOME NUMBER OF RANDOM VALUES FROM EACH OF THE STORMS USED!
if max_n_at_once > scramble_n:
max_n_at_once = scramble_n
end_val = 1
else:
end_val = len(range(scramble_n//max_n_at_once))-1
full_pre_median_list = []
full_post_median_list = []
for chunk in range(scramble_n//max_n_at_once):
if end_val!=1:
progressBar("Scrambling", value = chunk, endvalue=end_val)
pre_stack = np.ones(max_n_at_once)
post_stack = np.ones(max_n_at_once)
for key, dt_list in dts_dict.items():
random_choice_pres = np.random.choice(dt_list[0:-1],size = max_n_at_once) # cant use the last one as a pre-interval
random_choice_posts = np.random.choice(dt_list[1:], size = max_n_at_once) # cant use the first one as a pre-interval
random_choice_pres /= np.median(dt_list)
random_choice_posts /= np.median(dt_list)
pre_stack = np.vstack((pre_stack, random_choice_pres))
post_stack = np.vstack((post_stack, random_choice_posts))
#----------------------------------------------------------------------------------------------
# Remove the 1st row because it was a dummy
### EACH ROW = NEW STORM
### EACH COLUMN = NEW RANDOM FLASH
pre_stack = np.delete(pre_stack, 0, 0)
post_stack = np.delete(post_stack, 0, 0)
# Take median values
list_of_pre_medians = np.median(pre_stack, axis=0)
list_of_post_medians = np.median(post_stack, axis=0)
######
full_pre_median_list = np.concatenate((full_pre_median_list,list_of_pre_medians ))
full_post_median_list = np.concatenate((full_post_median_list, list_of_post_medians))
if scramble_n % max_n_at_once >0:
max_n_at_once = scramble_n % max_n_at_once
pre_stack = np.ones(max_n_at_once)
post_stack = np.ones(max_n_at_once)
for key, dt_list in dts_dict.items():
random_choice_pres = np.random.choice(dt_list[0:-1], size=max_n_at_once) # cant use the last one as a pre-interval
random_choice_posts = np.random.choice(dt_list[1:], size=max_n_at_once) # cant use the first one as a pre-interval
random_choice_pres /= np.median(dt_list)
random_choice_posts /= np.median(dt_list)
pre_stack = np.vstack((pre_stack, random_choice_pres))
post_stack = np.vstack((post_stack, random_choice_posts))
# ----------------------------------------------------------------------------------------------
# Remove the 1st row because it was a dummy
### EACH ROW = NEW STORM
### EACH COLUMN = NEW RANDOM FLASH
pre_stack = np.delete(pre_stack, 0, 0)
post_stack = np.delete(post_stack, 0, 0)
# Take median values
list_of_pre_medians = np.median(pre_stack, axis=0)
list_of_post_medians = np.median(post_stack, axis=0)
print(list_of_pre_medians)
######
full_pre_median_list = np.concatenate((full_pre_median_list, list_of_pre_medians))
full_post_median_list = np.concatenate((full_post_median_list, list_of_post_medians))
pre_hist, bins = np.histogram(full_pre_median_list, bins = np.arange(0.5, 1.5, 0.005)+0.0025)
post_hist, _ = np.histogram(full_post_median_list, bins =np.arange(0.5, 1.5, 0.005)+0.0025)
bin_centers =bins[0:-1]# + 0.5*(bins[1]-bins[0])
#pdffig = PdfPages('Plots/NON_TGF_SCRAMBLES.pdf')
plt.figure(figsize=(13, 5))
ax = plt.subplot(111)
plt.step(bin_centers, pre_hist, where='post',color='#354565', label = ' \'Pres\' ')
plt.step(bin_centers, post_hist, where='post', color = 'magenta',label = ' \'Posts\' ')
plt.axvline(x = 1.27, color='#354565', linestyle = '--')
plt.axvline(x = 0.97, color = 'magenta', linestyle = '--')
plt.title( "{} Trials\nNon-TGF-Producing".format(len(full_post_median_list)))
N_above = len(full_pre_median_list[full_pre_median_list>1.27])
N_below = len(full_post_median_list[full_post_median_list<0.97])
percent_above = N_above/scramble_n
percent_below = N_below/scramble_n
real_pre_med = 1.27
real_post_med = 0.97
plt.text(0.05, 0.5, "Percent above 1.27: {}, N = {}".format(round(percent_above*100,5), N_above),
color='#354565', transform=ax.transAxes, fontsize=12)
plt.text(0.05, 0.45,"Percent below 0.97: {}, N = {}".format(round(percent_below*100,5), N_below),
color = 'magenta', transform=ax.transAxes,fontsize=12)
plt.xlim(0.7, 1.3)
extraticks = [real_pre_med, real_post_med]
lim = ax.get_xlim()
ax.set_xticks(list(ax.get_xticks()) + extraticks)
ax.set_xlim(lim)
plt.xlabel("Ratio to Median")
plt.ylabel("# of Trials with Value")
plt.legend(loc = 'upper left')
if savefig_title != "":
plt.savefig(savefig_title+'_{}.pdf'.format(N_above))
plt.close()
else:
plt.show()
P.append(TheMaker(simulator, DNA, [SCV, Supply_Depot, Barracks, Refinery, Orbital_Command, MULE, Factory, Factory_Tech_Lab, Siege_Tank ]))
Q.append(TheMaker(simulator, DNA, [SCV, Supply_Depot, Barracks, Refinery, Barracks_Tech_Lab, Factory, Swap_Barracks_Factory_Tech_Lab, Siege_Tech, Siege_Tank]))
R.append(TheMaker(simulator, DNA, [SCV, Supply_Depot, Barracks, Refinery, Orbital_Command, Factory, Extra_Supplies, Siege_Tank ]))
S.append(TheMaker(simulator, DNA, [SCV, Refinery, Supply_Depot, Barracks, Barracks_Tech_Lab, Factory, Siege_Tank, Swap_Barracks_Factory_Tech_Lab, Siege_Tank, Siege_Tank]))
T.append(TheMaker(simulator, DNA, [SCV, Refinery, Supply_Depot, Barracks, Factory, Factory_Tech_Lab, Siege_Tech, Siege_Tank]))
U.append(TheMaker(simulator, DNA, [SCV, Refinery, Supply_Depot, Barracks, Factory, Factory_Tech_Lab, Siege_Tank, Siege_Tech ]))
#V.append(TheMaker(simulator, DNA, default))
#W.append(TheMaker(simulator, DNA, default))
#X.append(TheMaker(simulator, DNA, default))
#Y.append(TheMaker(simulator, DNA, default))
#Z.append(TheMaker(simulator, DNA, default))
Groups= [O, P, Q, R, S, T, U]
ct = count()
TotalSteps = 3000
pb = progressBar(minValue = 0, maxValue=TotalSteps)
i = ct.next()
pb(0)
past = datetime.now()
while i < TotalSteps:
pb(i)
if i > 9 and i % 10 == 0:
ETR = (datetime.now() - past).total_seconds()/3600./i*(TotalSteps - i)
EHR = int(ETR)
EMR = (ETR - EHR)*60
print '\n','; '.join(['%d, %d' % (max(g, key = lambda x: x.fitness()).fitness(), min(g, key = lambda x: x.fitness()).fitness()) for g in Groups]), 'ETR:%dh:%dm' % (EHR, EMR), '\n'
pb(i)
unique(Groups)
new = [SingleTournament(Lamarckican(g)) for g in Groups]
Groups = new
if i >=50 and i % 50 ==0: DoubleTournament(Groups)
def mcmc(Chain, runtime, mixingtime=1000, stepsize=1):
#mixingtime = 1000
#runtime = 50000
pb = progressBar(maxValue = runtime + mixingtime)
cwd=os.getcwd()
tmpdir = cwd+'/.'+uniquename()
if not tmpdir == None:
if os.path.exists(tmpdir):
os.chdir(tmpdir)
else:
os.mkdir(tmpdir)
os.chdir(tmpdir)
os.system('cp %s/%s %s/%s' % (cwd, parfile, tmpdir, parfile))
os.system('cp %s/%s %s/%s' % (cwd, pulsefile, tmpdir, pulsefile))
motifile(toafile, cwd, tmpdir)
MarkovChain = Chain
pf = PARfile(parfile)
#pf = model(parfile)
#pf.thawall()
pf.freezeall('DMX_0')
#pf.parameters['SINI'] = '0'
#pf.parameters['M2'] = '0'
#pf.parameters['XDOT'] = '0'
pf.write()
chisq, dof = tempofit(parfile, toafile = toafile, pulsefile = pulsefile)
#pf.tempofit(toafile, pulsefile = pulsefile)
chisq, dof = chisq, dof
pf.matrix(toafile)
#pf.freezeall()
#pf.thawall('JUMP_')
pf.write()
#if 'PAASCNODE'in pf.__dict__:
#plist = [x for x in pf.manifest if x in pf.parameters.keys()] + ['PAASCNODE']
#dict = {'BEST':[pf.__dict__[p][0] for p in plist[:-1]] + [pf.__dict__[p] for p in plist[-1:]], 'parfile':pf.parfile, 'parameters':plist + ['chisq']}
#else:
plist = [x for x in pf.manifest if x in pf.parameters.keys()]
dict = {'BEST':[pf.__dict__[p][0] for p in plist] + [pf.PAASCNODE, chisq], 'parfile':pf.parfile, 'parameters':plist + ['PAASCNODE', 'chisq']}
pickle.dump(dict, open('%s/bestpar.p' % cwd, 'w'), protocol=2)
p0 = probcal(pf)
p = p0
#print 'P0', p0
#try:
#MChain = pickle.load(open('MChain.p', 'r'))
#except:
#MChain = {'Chain':[]}
#MChain['parameters'] = plist
#pickle.dump(MChain, open('MChain.p', 'w'))
n = count()
m = count()
while n.next() <= mixingtime + runtime:
npf = pf.randomnew(stepsize=stepsize)
randomnew(npf, stepsize)
p1 = probcal(npf)
c = m.next()
if c % 30 == 0:pb(c)
if p1 > p0:
if c > mixingtime:
MarkovChain.append([npf.__dict__[p][0] for p in plist] + [npf.PAASCNODE, npf.chisq])
pf = npf
p0 = p1
if p1 > p:
p = p1
#if 'PAASCNODE' in plist:
#dict['BEST'] = [pf.__dict__[p][0] for p in plist[:-1]] + [pf.__dict__[p] for p in plist[-1:]] + [npf.chisq]
#else:
dict['BEST'] = [npf.__dict__[p][0] for p in plist] + [npf.PAASCNODE, npf.chisq]
pickle.dump(dict, open('%s/bestpar.p' % cwd, 'w'), protocol=2)
else:
t = uniform(0,1,1)[0]
if t < p1/p0:
if c > mixingtime:
MarkovChain.append([npf.__dict__[p][0] for p in plist] + [npf.PAASCNODE, npf.chisq])
pf = npf
p0 = p1
else:
if c > mixingtime:
MarkovChain.append([pf.__dict__[p][0] for p in plist] + [npf.PAASCNODE, npf.chisq])
#print MarkovChain
#print best
print '%d points added to the Chain.' % len(MarkovChain)
#OldChain = pickle.load(open('MChain.p','r'))['Chain']
#dict['Chain'] = OldChain + MarkovChain
#dict['Chain'] = MarkovChain
os.chdir(cwd)
def mcmc(Chain, runtime, mixingtime=1000):
#mixingtime = 1000
#runtime = 50000
pb = progressBar(maxValue = runtime)
cwd=os.getcwd()
tmpdir = cwd+'/'+uniquename()
if not tmpdir == None:
if os.path.exists(tmpdir):
os.chdir(tmpdir)
else:
os.mkdir(tmpdir)
os.chdir(tmpdir)
os.system('cp %s/%s %s/%s' % (cwd, parfile, tmpdir, parfile))
motifile(toafile, cwd, tmpdir)
dict = pickle.load(open('%s/bestpar.p' % cwd, 'r'))
Omega, cosi, m2 = dict['BEST']
MarkovChain = Chain
pf = PARfile(parfile)
pf.thawall()
pf.write()
pf.matrix(toafile)
pf.freezeall()
pf.write()
p0 = probcal(Omega, cosi, m2, pf)
p = p0
best = (Omega, cosi, m2 )
#dict = {'BEST':best}
#pickle.dump(dict, open('bestpar.p', 'w'))
n = count()
m = count()
while n.next() < mixingtime + runtime:
npf = pf.randomnew()
Ncosi = cosi + uniform(-0.05, 0.05)
#Nm2 = m2 + normal(0, 0.02)
NOmega = (Omega + normal(0, 5))
#Nsini = float(str(npf.SINI[0]))
#Ncosi = -1. * sqrt(1 - Nsini**2)
Nm2 = float(str(npf.M2[0]))
#print Ncosi, cosi, Ncosi - cosi, Nm2 - m2
p1 = probcal(NOmega, Ncosi, Nm2, npf)
c = m.next()
if p1 > p0:
if c > mixingtime:
MarkovChain.append((NOmega, Ncosi, Nm2 ))
pb(c - mixingtime)
pf = npf
p0 = p1
cosi, m2, Omega = Ncosi, Nm2, NOmega
if p1 > p:
best = (NOmega, Ncosi, Nm2)
p = p1
dict['BEST'] = best
pickle.dump(dict, open('%s/bestpar.p' % cwd, 'w'))
else:
t = uniform(0,1,1)[0]
if t < p1/p0:
if c > mixingtime:
MarkovChain.append((NOmega, Ncosi, Nm2 ))
pb(c - mixingtime)
pf = npf
p0 = p1
cosi, m2, Omega = Ncosi, Nm2, NOmega
else:
if c > mixingtime:
MarkovChain.append((Omega, cosi, m2))
#print MarkovChain
print best
print '%d new points generated.' %len(MarkovChain)
#OldChain = pickle.load(open('MChain.p','r'))['Chain']
#dict['Chain'] = OldChain + MarkovChain
#dict['Chain'] = MarkovChain
#os.rmdir(tmpdir)
os.chdir(cwd)