def testNoeud(n, p, listAlgo):
"""
Calcule le nombre de noeuds dans l'arbre de branchement visites par les
differents algorithmes de branchement passes dans listAlgo.
Args :
n : tableau numpy avec les valeurs de n a etre testes.
p : tableau numpy de meme taille que n avec les valeurs correspondants
de p a etre testes.
listAlgo : une liste d'algorithmes a etre testes. Elle doit etre une
liste de tuples. Dans chaque tuple, la premiere entree est une chaine
de caracteres contenant le nom de la methode utilisee et la deuxieme
entree doit etre un dictionnaire avec les arguments passes a la methode.
Les cles de ce dictionnaire sont les noms des arguments (chaines de
caracteres) et les valeurs sont les valeurs des arguments.
Returns :
res : tableau numpy de taille len(listAlgo) x n.size contenant, en
ligne i et colonne j, le nombre de noeuds visites par l'algorithme
listAlgo[i] dans un graphe aleatoire a n[j] sommets et probabilite
d'aretes p[j].
"""
assert n.size == p.size
res = np.zeros((len(listAlgo), n.size), dtype = int)
progressBar = ProgressBar(maxValue = n.size * len(listAlgo))
for ni in range(n.size):
for i, (algoName, algoArgs) in enumerate(listAlgo):
progressBar.update(ni * len(listAlgo) + i + 1)
graphe = Graphe(nbSommets = n[ni], probaArete = p[ni])
_, noeuds = getattr(graphe, algoName)(**algoArgs)
res[i, ni] = noeuds
print("")
return res
def test():
p = int(input("Enter file iteration: "))
fileName = "3D Maps/3D Matrix(" + str(p) + ").txt"
x = 0
count = 0
no = 0
temp = []
pbar = ProgressBar(getFileSize(fileName), "Reading in...")
file = open(fileName)
for line in file:
x += 1
if x > 2:
vals = line.rstrip().split(",")
for element in vals:
if element == "None" or element == None:
no += 1
pbar.update()
count += len(vals)
else:
pbar.update()
file.close()
print()
print("Lines in file............= ", x)
print("Elements in file.........= ", count)
print("Number of None types.....= ", no)
print("Number of not None types.= ", count - no)
a = input("**Hit ENTER to close**") #keeps the terminal open
return
def main():
fileName = sys.argv[1]
fileH = MY_FILEH(fileName)
syshostPattern = re.compile(r'syshost:([^ ]*) ')
fnSz = os.path.getsize(fileName)
pbar = ProgressBar(maxVal=fnSz)
sz = 0
t1 = time.time()
with open(fileName) as f:
for line in f:
sz += len(line)
pbar.update(sz)
m = syshostPattern.search(line)
if (not m):
continue
syshost = m.group(1)
fh = fileH.get_file_handle(syshost)
fh.write(line)
fileH.close_file_handles()
pbar.fini()
t2 = time.time()
print("Time: ", time.strftime("%T", time.gmtime(t2 - t1)))
def trainingHelper(self, trainingData, trainingLabels, iterations):
"""
trainingHelper() classifies training data using the perceptron weights
and updates the perceptron weights if the perceptron is incorrect.
Keyword Arguments:
trainingData -- training data for the perceptron
trainingLabels -- labels for the associated training data
iterations -- desired number of iterations over the training dataset.
"""
for i in range(iterations):
progressBar = ProgressBar(100, len(trainingData), "Learning Weights, Iteration {0} of {1}"
.format(i + 1, iterations))
for j in range(len(trainingData)):
progressBar.update(j)
values = util.Counter()
# Go over each label, and create the value from the training data and current vectors
for label in self.legalLabels:
activation = 0
for key, value in trainingData[j].items():
activation += self.weights[label][key] * value
values[label] = activation
# Here, we update values in weight vectors if we reach an incorrect conclusion.
if values.argMax() != trainingLabels[j]:
for key, value in trainingData[j].items():
self.weights[trainingLabels[j]][key] += value
self.weights[values.argMax()][key] -= value
progressBar.clear()
def testTempsPfunc(nMax, p, nomMethode, fois = 10, **kwargs):
"""
Teste le temps d'execution moyen d'un algorithme pour le probleme de
couverture de graphe passe par l'argument nomMethode. Son fonctionnement
est identique a la fonction testTempsN, a part le fait que l'argument p est
maintenant une fonction de n.
Args :
nMax : valeur maximale du nombre de sommets du graphe.
p : fonction prennant en argument une valeur de n et renvoyant la
probabilite de presence d'une arete pour cette valeur de n.
nomMethode : chaine de caracteres donnant le nom de la methode a tester.
fois (facultatif) : nombre de repetitions pour chaque valeur de n et p.
**kwargs (facultatif) : arguments passes a la methode a tester.
Returns :
res : tableau numpy de taille 10 avec les temps moyens d'execution en
secondes pour chaque n.
n : tableau numpy de taille 10 contenant les valeurs de n utilises
dans les tests.
"""
n = np.linspace(nMax / 10, nMax, 10, dtype = int)
res = np.zeros(n.shape)
progressBar = ProgressBar(maxValue = n.size * fois)
for ni in range(n.size):
for f in range(fois):
progressBar.update(ni * fois + f + 1)
graphe = Graphe(nbSommets = n[ni], probaArete = p(n[ni]))
start = t.process_time()
getattr(graphe, nomMethode)(**kwargs)
res[ni] += t.process_time() - start
print("")
return res / fois, n
def main():
fileName = sys.argv[1]
fileH = MY_FILEH(fileName)
syshostPattern = re.compile(r'syshost:([^ ]*) ')
fnSz = os.path.getsize(fileName)
pbar = ProgressBar(maxVal=fnSz)
sz = 0
t1 = time.time()
with open(fileName) as f:
for line in f:
sz += len(line)
pbar.update(sz)
m = syshostPattern.search(line)
if (not m):
continue
syshost = m.group(1)
fh = fileH.get_file_handle(syshost)
fh.write(line)
fileH.close_file_handles()
pbar.fini()
t2 = time.time()
print("Time: ", time.strftime("%T", time.gmtime(t2-t1)))
def classify(self, data):
"""
classify() classifies each data item in the input by finding the best prototype vector.
Keyword Arguments:
data -- the test data to classify
"""
guesses = []
progressBar = ProgressBar(100, len(data), "Classifying Data")
for index, entry in enumerate(data):
progressBar.update(index)
values = util.Counter()
# for each label, compute activation
for label in self.legalLabels:
activation = 0
# sum over the weights * values to get activation
for key, value in entry.items():
activation += self.weights[label][key] * value
values[label] = activation
# add classification guess for data by getting the argmax
guesses.append(values.argMax())
progressBar.clear()
return guesses
def maxMatrix(it):
i = 0
lineNum = 0
altMax = None
altMin = None
size = getFileSize(it)
pbar = ProgressBar(size, "Reading in data...")
#reads in the data from the file
file = open(it)
for line in file:
points = []
vals = line.rstrip().split(',')
#determine the max altitude
for element in vals:
if altMin == None and float(element) > 2.4384:
altMin = float(element)
elif altMin != None and float(element) < altMin and float(
element) >= 2.4384:
altMin = float(element)
if altMax == None:
altMax = float(element)
elif float(element) > altMax:
altMax = float(element)
lineNum = i
i += 1
pbar.update()
file.close()
return altMax, altMin
def searchForBestAccuracy(y, pred, tries=100, step=1):
pb = ProgressBar(len(range(1, tries, step)))
accs = [calculateAccuracy(y, pred)]
x = [0]
bestwindow = 0
bestacc = 0.0
for window in xrange(1, tries, step):
f = Filter(pred, window)
newp = []
for target, lwin in f:
dc = Counter(lwin)
target = max(dc.iteritems(), key=operator.itemgetter(1))[0]
newp.append(target)
accuracy = calculateAccuracy(y, newp)
accs.append(accuracy)
if accuracy > bestacc:
bestacc = accuracy
bestwindow = window
pb.update()
x.extend(range(1, tries, step))
#print len(x), len(accs)
plt.plot(x, accs)
plt.savefig('accurLocal.png', bbox_inches='tight')
print
print 'Best window:', bestwindow
print 'Best accuracy:', bestacc
return bestwindow
def main():
"""
Walks the list of users via the passwd_generator and load the
link and run files.
"""
# Push command line on to XALT_Stack
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"' + v + '"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(dbConfigFn(args.dbname))
num = int(capture("getent passwd | wc -l"))
pbar = ProgressBar(maxVal=num)
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
iuser = 0
lnkCnt = 0
runCnt = 0
xaltDir = args.datadir
if (os.path.isdir(xaltDir)):
iuser += 1
linkFnA = files_in_tree(xaltDir, "*/link.*.json")
XALT_Stack.push("link_json_to_db()")
lnkCnt += link_json_to_db(xalt, args.listFn, rmapT, linkFnA)
XALT_Stack.pop()
if (args.delete):
remove_files(linkFnA)
#remove_files(files_in_tree(xaltDir, "*/.link.*.json"))
runFnA = files_in_tree(xaltDir, "*/run.*.json")
XALT_Stack.push("run_json_to_db()")
runCnt += run_json_to_db(xalt, args.listFn, rmapT, runFnA)
XALT_Stack.pop()
if (args.delete):
remove_files(runFnA)
#remove_files(files_in_tree(xaltDir, "*/.run.*.json"))
icnt += 1
pbar.update(icnt)
xalt.connect().close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("num users: ", iuser, ", num links: ", lnkCnt, ", num runs: ",
runCnt)
def main():
"""
Walks the list of users via the passwd_generator and load the
link and run files.
"""
# Push command line on to XALT_Stack
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"'+v+'"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(dbConfigFn(args.dbname))
num = int(capture("getent passwd | wc -l"))
pbar = ProgressBar(maxVal=num)
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
iuser = 0
lnkCnt = 0
runCnt = 0
xaltDir = args.datadir
if (os.path.isdir(xaltDir)):
iuser += 1
linkFnA = files_in_tree(xaltDir, "*/link.*.json")
XALT_Stack.push("link_json_to_db()")
lnkCnt += link_json_to_db(xalt, args.listFn, rmapT, linkFnA)
XALT_Stack.pop()
if (args.delete):
remove_files(linkFnA)
#remove_files(files_in_tree(xaltDir, "*/.link.*.json"))
runFnA = files_in_tree(xaltDir, "*/run.*.json")
XALT_Stack.push("run_json_to_db()")
runCnt += run_json_to_db(xalt, args.listFn, rmapT, runFnA)
XALT_Stack.pop()
if (args.delete):
remove_files(runFnA)
#remove_files(files_in_tree(xaltDir, "*/.run.*.json"))
icnt += 1
pbar.update(icnt)
xalt.connect().close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("num users: ", iuser, ", num links: ", lnkCnt, ", num runs: ", runCnt)
def generateActions(backupDataSet, config):
inNewDir = None
actions = []
progbar = ProgressBar(50, 1000, len(backupDataSet.fileDirSet))
for i, element in enumerate(backupDataSet.fileDirSet):
progbar.update(i)
# source\compare
if element.inSourceDir and not element.inCompareDir:
stats.files_to_copy += 1
stats.bytes_to_copy += element.fileSize
if inNewDir != None and element.path.startswith(inNewDir):
actions.append(Action("copy", element.isDirectory, name=element.path, htmlFlags="inNewDir"))
else:
if element.isDirectory:
inNewDir = element.path
actions.append(Action("copy", True, name=element.path, htmlFlags="Folder"))
else:
actions.append(Action("copy", False, name=element.path))
# source&compare
elif element.inSourceDir and element.inCompareDir:
if element.isDirectory:
if config["versioned"] and config["compare_with_last_backup"]:
# Formerly, only empty directories were created. This step was changed, as we want to create all directories
# explicitly for setting their modification times later
if dirEmpty(os.path.join(backupDataSet.sourceDir, element.path)):
actions.append(Action("copy", True, name=element.path, htmlFlags="emptyFolder"))
else:
actions.append(Action("copy", True, name=element.path, htmlFlags="Folder"))
else:
# same
if filesEq(os.path.join(backupDataSet.sourceDir, element.path), os.path.join(backupDataSet.compareDir, element.path), config["compare_method"]):
if config["mode"] == "hardlink":
actions.append(Action("hardlink", False, name=element.path))
stats.files_to_hardlink += 1
stats.bytes_to_hardlink += element.fileSize
# different
else:
actions.append(Action("copy", False, name=element.path))
stats.files_to_copy += 1
stats.bytes_to_copy += element.fileSize
# compare\source
elif not element.inSourceDir and element.inCompareDir:
if config["mode"] == "mirror":
if not config["compare_with_last_backup"] or not config["versioned"]:
actions.append(Action("delete", element.isDirectory, name=element.path))
stats.files_to_delete += 1
stats.bytes_to_delete += element.fileSize
print("") # so the progress output from before ends with a new line
return actions
def main():
args = argument_parser()
progress_bar = ProgressBar(args.input)
date = parse_date(args.date)
input = open(args.input, 'r')
output = open(args.output, 'w')
if args.zmap_log:
log_dict = zmap_log(args.port, date, input)
output.write(json.dumps(log_dict))
sys.exit(0)
progress_bar.start()
for line in input:
progress_bar.update(1)
data = json.loads(line)
if args.clean_errors and clean_json(data):
continue
if args.dns_reverse:
data = dns_reverse(data)
if args.whois:
data = ip_whois(data)
if args.http:
data = http_protocol(data, args.old_data)
if args.https:
data = https_protocol(data, date)
if args.ssh:
data = ssh_protocol()
data['date'] = date.strftime("%Y-%m-%d")
data['schema_version'] = '1.0'
try:
output.write(json.dumps(data) + '\n')
except UnicodeDecodeError:
output.write(json.dumps(data, encoding='latin1') + '\n')
progress_bar.finish()
def makeMatrix(y,x):
rowArray = []
matrix = []
i = 0
j = 0
pbar = ProgressBar(y,"Creating Matrix...")
#creates the appropriately sized rowArray
while i < y:
j = 0
rowArray = []
while j < x:
rowArray.append(None)
j = j + 1
matrix.append(rowArray)
i = i + 1
pbar.update()
return matrix
def readFile(it):
temp = []
matrix = []
fileT = "./3D Maps/3D Matrix("+str(it)+").txt"
size = getFileSize(fileT)
heading = "Reading in matrix("+str(it)+")..."
pbar = ProgressBar(size,heading)
#reads in the data from the file
file = open(fileT)
for line in file:
temp = []
vals = line.rstrip().split(',')
for element in vals:
temp.append(float(element))
matrix.append(temp)
pbar.update()
return matrix
def convert_save(self, filename):
widgets = [' ', Bar('#'), ' ', BarLabel('Parsing "%s" ....' % filename, 'Parsing "%s" Done' % filename), ' | ',Percentage(),' | ', ETA(), ' ']
tmpname = filename + '.' + self.convertTo
fr = open(filename)
fw = open(tmpname, 'w')
bar = ProgressBar(widgets=widgets, maxval=os.path.getsize(filename)).start()
try:
while True:
line = fr.readline()
if not line: break
fw.write( self.__parse(line) )
bar.update(fr.tell())
shutil.move(tmpname, filename)
bar.finish()
finally:
fr.close()
fw.close()
def main():
args = CmdLineOptions().execute()
xalt = XALTdb(ConfigFn)
num = int(capture("getent passwd | wc -l"))
pbar = ProgressBar(maxVal=num)
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
iuser = 0
lnkCnt = 0
runCnt = 0
for user, hdir in passwd_generator():
xaltDir = os.path.join(hdir, ".xalt.d")
if (os.path.isdir(xaltDir)):
iuser += 1
linkFnA = files_in_tree(xaltDir, "*/link.*.json")
lnkCnt += link_json_to_db(xalt, user, rmapT, linkFnA)
if (args.delete):
remove_files(linkFnA)
remove_files(files_in_tree(xaltDir, "*/.link.*.json"))
runFnA = files_in_tree(xaltDir, "*/run.*.json")
runCnt += run_json_to_db(xalt, user, rmapT, runFnA)
if (args.delete):
remove_files(runFnA)
remove_files(files_in_tree(xaltDir, "*/.run.*.json"))
icnt += 1
pbar.update(icnt)
xalt.connect().close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("num users: ", iuser, ", num links: ", lnkCnt, ", num runs: ",
runCnt)
def main():
altMax = None
altMin = None
fileName = input("Enter file name: ")
fileName = "./Drone Data/" + fileName
it = (input("Enter iteration: "))
it = "./3D Maps/3D Matrix(" + it + ").txt"
size = getFileSize(fileName)
pbar = ProgressBar(size, "Reading in data...")
#reads in the data from the file
file = open(fileName)
for line in file:
points = []
vals = line.rstrip().split(',')
for element in vals:
temp = element.split(' ')
points.append(temp[len(temp) - 1])
vals = points
#determine the max altitude
if altMax == None:
altMax = abs(float(vals[2]))
altMin = abs(float(vals[2]))
elif abs(float(vals[2])) > altMax:
altMax = abs(float(vals[2]))
elif abs(float(vals[2])) < altMin:
altMin = abs(float(vals[2]))
pbar.update() #updates the prgress bar
file.close() #closes the file
maxM, minM = maxMatrix(it)
print("Max altitude in matrix.= ", maxM)
print("Min altitude in matrix.= ", minM)
print("Max altitude...........= ", altMax)
print("Min altitude...........= ", altMin)
a = input("**Hit ENTER to close**") #keeps the terminal open
return
def main():
args = CmdLineOptions().execute()
xalt = XALTdb(ConfigFn)
num = int(capture("getent passwd | wc -l"))
pbar = ProgressBar(maxVal=num)
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
iuser = 0
lnkCnt = 0
runCnt = 0
for user, hdir in passwd_generator():
xaltDir = os.path.join(hdir,".xalt.d")
if (os.path.isdir(xaltDir)):
iuser += 1
linkFnA = files_in_tree(xaltDir, "*/link.*.json")
lnkCnt += link_json_to_db(xalt, user, rmapT, linkFnA)
if (args.delete):
remove_files(linkFnA)
remove_files(files_in_tree(xaltDir, "*/.link.*.json"))
runFnA = files_in_tree(xaltDir, "*/run.*.json")
runCnt += run_json_to_db(xalt, user, rmapT, runFnA)
if (args.delete):
remove_files(runFnA)
remove_files(files_in_tree(xaltDir, "*/.run.*.json"))
icnt += 1
pbar.update(icnt)
xalt.connect().close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("num users: ", iuser, ", num links: ", lnkCnt, ", num runs: ", runCnt)
def classify(self, testData):
"""
classify() classifies each data item in the input by finding the best prototype vector.
Keyword Arguments:
testData -- the test data to classify
"""
guesses = []
self.posteriors = [
] # Log posteriors are stored for later data analysis.
counter = 0
size = len(testData)
progressBar = ProgressBar(100, len(testData), "Classifying Data")
for index, datum in enumerate(testData):
progressBar.update(index)
posterior = self.calculateLogJointProbabilities(datum)
guesses.append(posterior.argMax())
self.posteriors.append(posterior)
progressBar.clear()
return guesses
def backgroundSubtraction(inputfile):
"""
Receives a file containing paths to images in jpg format labels.
The background is subtracted from each image and the new image is
saved into `outfolder`. A new path file is generated containing the
path to the image without the background and its respective label.
Parameters:
-----------
input_folder : string
path to the folder containing images
Notes:
------
The dictionary generated by `pathfiles.genDictFromPaths()` has the form:
dic {id_data: {activity: [(img, class)]}}
"""
fout = open(join(PATH, 'pathbg_label.txt'), 'w')
dic, n = pathfiles.genDictFromPaths(inputfile)
pb = ProgressBar(n)
for data in sorted(dic):
for activity in dic[data]:
imgs = sorted(dict(dic[data][activity]).keys())
fgbg = cv2.BackgroundSubtractorMOG2()
fgbg = trainBackground(fgbg, data, activity, imgs[:100])
for idimg, y in sorted(dic[data][activity]):
path_img = join(PATH, 'data' + str(data), activity,
'img' + str(SIZE_IMG),
str(idimg) + '.jpg')
dirbg = join(PATH, 'data' + str(data), activity,
'bg' + str(SIZE_IMG))
if not isdir(dirbg):
os.mkdir(dirbg)
path_out = join(dirbg, str(idimg) + '.jpg')
img = cv2.imread(path_img, 1)
fgmask = fgbg.apply(img)
cv2.imwrite(path_out, fgmask)
fout.write('%s %d\n' % (path_out, y))
pb.update()
def testEcart(nMax, p, nomMethode, fois = 10):
"""
Teste l'ecart entre les solutions donnees par les algorithmes approches et
la solution exacte. L'algorithme approche a etre teste est donne par
l'argument nomMethode, alors que l'algorithme exact de comparaison est
l'algorithme de branchement ameliore avec sommets pris par ordre
decroissant de degre et elimination de sommets de degre 1.
La fonction cree des graphes aleatoires a n sommets pour 10 valeurs de n
allant de nMax/10 jusqu'à nMax, avec un choix aleatoire de presence
d'aretes donne par la probabilite constante p. Pour chaque n et p, le temps
est calcule par une moyenne sur fois graphes.
Args :
nMax : valeur maximale du nombre de sommets du graphe.
p : probabilite de presence d'une arete, constante.
nomMethode : chaine de caracteres donnant le nom de la methode a tester.
fois (facultatif) : nombre de repetitions pour chaque valeur de n et p.
Returns :
res : tableau numpy de taille 10 x fois avec, en ligne i et colonne j,
le rapport entre la longueur de la solution trouvee par nomMethode
et la solution exacte avec un graphe a n[i] sommets et lors de
l'execution j.
n : tableau numpy de taille 10 contenant les valeurs de n utilises
dans les tests.
"""
n = np.linspace(nMax / 10, nMax, 10, dtype = int)
res = np.zeros((n.size, fois))
progressBar = ProgressBar(maxValue = n.size * fois)
for ni in range(n.size):
for f in range(fois):
progressBar.update(ni * fois + f + 1)
graphe = Graphe(nbSommets = n[ni], probaArete = p)
resMethode = getattr(graphe, nomMethode)()
resExacte, _ = graphe.algoBranchementAmeliore(sommetMax = True, elimDegre1 = True)
res[ni, f] = len(resMethode) / len(resExacte) if len(resExacte) != 0 else 1
print("")
return res, n
def findAllMissing(
table,
acc=0): #acc = 0 is normal accuracy acc = 1 is high accuracy
pos = 0
size = len(table)
if acc == 1:
size = size * 2
pbar = ProgressBar(size, "Packing Matrix...")
while pos < len(table):
table[pos] = solveMissing(table, pos)
pos = pos + 1
pbar.update()
#high accuracy
if acc == 1:
pos = 0
while pos < len(table):
table[pos] = solveMissing(table, pos)
pos = pos + 1
pbar.update()
return table
def defineFeatures(imageList, chop):
"""
defineFeatures() defines a simple feature of a pixel either being white (0)
or not (1) for a list of images and pixel values
chops off pixels on outside of image for faster (but less accurate) classification
"""
featureList = []
features = []
progressBar = ProgressBar(100, len(imageList),
"Getting Features for Images")
for index, image in enumerate(imageList):
# update progress
progressBar.update(index)
# create feature of on/off for (x, y) positions in image
imgFeature = Counter()
for x in range(chop, len(image) - chop):
for y in range(chop, len(image[x]) - chop):
if image[x][y] == 0:
imgFeature[(x, y)] = 0
else:
imgFeature[(x, y)] = 1
featureList.append(imgFeature)
progressBar.clear()
# Here, we create a list of all of the features for use in the
# perceptron and Naive Bayes classifiers.
if len(imageList) > 0:
image = imageList[0]
for x in range(chop, len(image) - chop):
for y in range(chop, len(image[x]) - chop):
features.append((x, y))
return featureList, features
def testTempsN(nMax, p, nomMethode, fois = 10, **kwargs):
"""
Teste le temps d'execution moyen d'un algorithme pour le probleme de
couverture de graphe passe par l'argument nomMethode.
Pour cela, la fonction cree des graphes aleatoires a n sommets pour 10
valeurs de n allant de nMax/10 jusqu'a nMax, avec un choix aleatoire de
presence d'aretes donne par la probabilite constante p.
Pour chaque n et p, le temps est calcule par une moyenne sur fois graphes.
Args :
nMax : valeur maximale du nombre de sommets du graphe.
p : probabilité de présence d'une arete, constante.
nomMethode : chaine de caracteres donnant le nom de la méthode a tester.
fois (facultatif) : nombre de repetitions pour chaque valeur de n et p.
**kwargs (facultatif) : arguments passes a la methode a tester.
Returns :
res : tableau numpy de taille 10 avec les temps moyens d'execution
en secondes pour chaque n.
n : tableau numpy de taille 10 contenant les valeurs de n utilises
dans les tests.
"""
n = np.linspace(nMax / 10, nMax, 10, dtype = int)
res = np.zeros(n.shape)
progressBar = ProgressBar(maxValue = n.size * fois)
for ni in range(n.size):
for f in range(fois):
progressBar.update(ni * fois + f + 1)
graphe = Graphe(nbSommets = n[ni], probaArete = p)
start = t.process_time()
getattr(graphe, nomMethode)(**kwargs)
res[ni] += t.process_time() - start
print("")
return res / fois, n
def searchForBest(inputfile, tries=100, step=2):
paths, ys, preds = [], [], []
with open(inputfile) as fin:
for n, line in enumerate(fin):
path, y, pred = line.strip().split()
paths.append(path)
ys.append(y)
preds.append(pred)
pb = ProgressBar(len(range(1, tries, step)))
accs = [calculateAccuracy(ys, preds)]
x = [0]
bestwindow = 0
bestacc = 0.0
for window in xrange(1, tries, step):
f = Filter(preds, window)
newp = []
for target, lwin in f:
dc = Counter(lwin)
target = max(dc.iteritems(), key=operator.itemgetter(1))[0]
newp.append(target)
accuracy = calculateAccuracy(ys, newp)
accs.append(accuracy)
if accuracy > bestacc:
bestacc = accuracy
bestwindow = window
pb.update()
x.extend(range(1, tries, step))
#print len(x), len(accs)
plt.plot(x, accs)
plt.savefig('accur.png', bbox_inches='tight')
print
print 'Best window:', bestwindow
print 'Best accuracy:', bestacc
return bestwindow
def main():
"""
read from syslog file into XALT db.
"""
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"'+v+'"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(ConfigFn)
syslogFile = args.syslog
# should add a check if file exists
num = int(capture("cat "+syslogFile+" | wc -l"))
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
lnkCnt = 0
runCnt = 0
badCnt = 0
count = 0
if num != 0:
pbar = ProgressBar(maxVal=num)
if (os.path.isfile(syslogFile)):
f=open(syslogFile, 'r')
for line in f:
if "XALT_LOGGING" in line:
i=line.find("link:")
if i == -1 :
i=line.find("run:")
if i == -1:
continue # did not find a link or run, continue to next line
array=line[i:].split(":")
type = array[0].strip()
syshost = array[1].strip()
try:
resultT=json.loads(base64.b64decode(array[2]))
XALT_Stack.push("XALT_LOGGING: " + type + " " + syshost)
# XALT_Stack.push("XALT_LOGGING resultT: " + resultT)
if ( type == "link" ):
XALT_Stack.push("link_to_db()")
xalt.link_to_db(rmapT, resultT)
XALT_Stack.pop()
lnkCnt += 1
elif ( type == "run" ):
XALT_Stack.push("run_to_db()")
xalt.run_to_db(rmapT, resultT)
XALT_Stack.pop()
runCnt += 1
else:
print("Error in xalt_syslog_to_db")
XALT_Stack.pop()
except:
badCnt += 1
# figure out length of array[2], as it might be
# near the syslog message size limit
strg=array[2]
lens = len(strg)
# lenx = lens - (lens % 4 if lens % 4 else 4)
print("xalt_syslog_to_db: undecodable entry! length: ",lens)
# resultT = base64.decodestring(strg[:lenx])
# print("result: ",result)
count += 1
pbar.update(count)
pbar.fini()
# what should be done if there are errors?
# what went wrong?
# how do we restart?
#
# xalt.connect().close()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
#
print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ", runCnt, ", badCnt: ", badCnt)
def trainAndTune(self, trainingData, trainingLabels, validationData,
validationLabels, kgrid):
"""
trainAndTune() trains the classifier by collecting counts over the training data and choosing the smoothing parameter among the choices in kgrid by
using the validation data. This method should store the right parameters
as a side-effect and should return the best smoothing parameters.
Keyword Arguments:
trainingData -- training data for the perceptron
trainingLabels -- labels for the associated training data
validationData -- validation data for the perceptron tuning function
validationLabels -- labels for the associated validation data
kgrid -- a list of possible k values to try for smoothing
"""
# We begin by creating the prior probabilities for each of the labels
# and the features based on the counts in the training data.
countLabel = util.Counter() # in form k = label, v = numOfL
countFeature = util.Counter()
# We begin looking over the training data here.
progressBar = ProgressBar(100, len(trainingData), "Counting Data")
for i in range(len(trainingData)):
# update our progress bar
progressBar.update(i)
label = trainingLabels[i]
# Labels are counted at each point they are seen here.
countLabel[label] += 1
# Then, if we haven't seen the label, we add it to the feature counter.
if label not in countFeature:
countFeature[label] = util.Counter()
# Finally, we loop over the features for each datum and add each feature once
# for each occurrence.
for feature in trainingData[i]:
countFeature[label][feature] += trainingData[i][feature]
progressBar.clear()
self.probLabel = copy.deepcopy(countLabel)
self.probLabel.normalize()
# At this point we have the counts, and we want to see what level of smoothing
# increases our accuracy the most over the training set. Essentially, we just
# create all of the probabilities from the feature counts while adding the smoothing
# and classify the training data each time and pick whatever was most accurate.
kClassifications = util.Counter()
probForK = util.Counter()
numCorrectK = util.Counter()
print "Validation Accuracy"
print "==================="
for k in kgrid:
# make counter for probabilities for each k
probForK[k] = util.Counter()
# make counters for probabilities for each label
for label in self.legalLabels:
probForK[k][label] = util.Counter()
# find probability of each feature given each label
progressBar = ProgressBar(
100, len(self.features),
"Getting Probabilities for Features, Label {0}".format(
label))
for index, feature in enumerate(self.features):
progressBar.update(index)
if countFeature[label] != 0:
probForK[k][label][feature] = float(
countFeature[label][feature] +
k) / (countLabel[label] + k * len(self.features))
progressBar.clear()
# set probabilities for features and classify validation data
self.probFeature = probForK[k]
classificationLabels = self.classify(validationData)
# check how much of the data was classified correctly
correct = 0
for i in range(len(classificationLabels)):
if classificationLabels[i] == validationLabels[i]:
correct += 1
# print accuracy for each k
print "k = {0}, number of correct classifications = {1}".format(
k, correct)
# store the number of correct classifications for k value
numCorrectK[k] = correct
# pick k from our list of possible k values
self.k = None
for k in numCorrectK:
# find k with the highest number of correct classifications
# if there is a tie, use a lower k value
if (self.k == None or numCorrectK[self.k] < numCorrectK[k]
or (numCorrectK[self.k] == numCorrectK[k] and k < self.k)):
self.k = k
self.probFeature = probForK[self.k]
# print final choice for k
print "K chosen = {0}".format(self.k)
return self.k
def executeActionList(dataSet):
logging.info("Applying actions for the target \"" + dataSet.name + "\"")
if len(dataSet.actions) == 0:
logging.warning("There is nothing to do for the target \"" +
dataSet.name + "\"")
return
os.makedirs(dataSet.targetDir, exist_ok=True)
progbar = ProgressBar(50, 1000, len(dataSet.actions))
# Phase 1: apply the actions
for i, action in enumerate(dataSet.actions):
progbar.update(i)
actionType = action["type"]
params = action["params"]
try:
if actionType == "copy":
fromPath = os.path.join(dataSet.sourceDir, params["name"])
toPath = os.path.join(dataSet.targetDir, params["name"])
logging.debug('copy from "' + fromPath + '" to "' + toPath +
'"')
#TODO: remove the manual checks for isFile etc., switch to action["isDir"]
if os.path.isfile(fromPath):
os.makedirs(os.path.dirname(toPath), exist_ok=True)
shutil.copy2(fromPath, toPath)
stats.bytes_copied += os.path.getsize(
fromPath
) # If copy2 doesn't fail, getsize shouldn't either
stats.files_copied += 1
elif os.path.isdir(fromPath):
os.makedirs(toPath, exist_ok=True)
else:
# We know there is a problem, because isfile and isdir both return false. Most likely permissions or a missing file,
# in which case the error handling is done in the permission check. If not, throw a general error
accessible, _ = filesize_and_permission_check(fromPath)
if accessible:
logging.error(
"Entry \"" + fromPath +
"\" exists but is neither a file nor a directory.")
stats.backup_errors += 1
elif actionType == "delete":
path = os.path.join(dataSet.targetDir, params["name"])
logging.debug('delete file "' + path + '"')
stats.files_deleted += 1
if os.path.isfile(path):
stats.bytes_deleted += os.path.getsize(path)
os.remove(path)
elif os.path.isdir(path):
shutil.rmtree(path)
elif actionType == "hardlink":
fromPath = os.path.join(dataSet.compareDir, params["name"])
toPath = os.path.join(dataSet.targetDir, params["name"])
logging.debug('hardlink from "' + fromPath + '" to "' +
toPath + '"')
toDirectory = os.path.dirname(toPath)
os.makedirs(toDirectory, exist_ok=True)
hardlink(fromPath, toPath)
stats.bytes_hardlinked += os.path.getsize(
fromPath
) # If hardlink doesn't fail, getsize shouldn't either
stats.files_hardlinked += 1
else:
logging.error("Unknown action type: " + actionType)
except Exception as e:
logging.error(e)
stats.backup_errors += 1
print("") # so the progress output from before ends with a new line
# Phase 2: Set the modification timestamps for all directories
# This has to be done in a separate step, as copying into a directory will reset its modification timestamp
logging.info(
"Applying directory modification timestamps for the target \"" +
dataSet.name + "\"")
progbar.update(0)
for i, action in enumerate(dataSet.actions):
progbar.update(i)
params = action["params"]
if not action["isDir"]:
continue
try:
fromPath = os.path.join(dataSet.sourceDir, params["name"])
toPath = os.path.join(dataSet.targetDir, params["name"])
logging.debug('set modtime for "' + toPath + '"')
modTime = os.path.getmtime(fromPath)
os.utime(toPath, (modTime, modTime))
except Exception as e:
logging.error(e)
stats.backup_errors += 1
print("") # so the progress output from before ends with a new line
qt = os.listdir(weights_path)
pb = ProgressBar(len(qt) / 2)
for k, weight_file in enumerate(sorted(qt)):
if "caffemodel" not in weight_file:
continue
iteration = re.findall("\d+", weight_file)[-1]
if iteration in tested_models:
# print iteration, "model already tested. Skip this model!"
continue
# print "-----> Processing model", iteration, "<-----"
#update_progress((k+1)/2,len(qt)/2)
pb.update()
caffe_weights = "-weights " + os.path.join(weights_path,
weight_file) + " "
command = caffe_command + caffe_model + caffe_weights + caffe_gpu + caffe_iterations
output_file.write("Testing model " + weight_file + "\n")
p = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=True)
loss = []
accuracy = []
while True:
line = p.stdout.readline()
if not line:
break
line = line.strip()
def main():
ACCURACY = 1 #controls the accuracy when solving for missing data points
#1 = high accuracy
#0 = standard accuracy
longs = set() #unique longitudes
latMin = None #minimum latitudes
longMin = None #minimum longitude
latMax = None #maximum latitude
longMax = None #maximum longitude
altMax = None #maximum altitude
x = 0 #current line number
size = 0
interval = float(input("Enter the inches per pixel: "))
fileName = input("Enter file name: ")
fileName = "./Drone Data/" + fileName
size = getFileSize(fileName)
pbar = ProgressBar(size,"Reading in data...")
#reads in the data from the file
file = open(fileName)
for line in file:
points = []
vals = line.rstrip().split(',')
#for element in vals:
#temp = element.split(' ')
#points.append(temp[len(temp)-1])
#vals = points
longs.add(round(float(vals[1]),2)) #places unique longs in a set
#determines the max and min latitudes
if latMax == None:
latMax = abs(round(float(vals[0]),2))
latMin = abs(round(float(vals[0]),2))
elif abs(round(float(vals[0]),2)) > latMax:
latMax = abs(round(float(vals[0]),2))
elif abs(round(float(vals[0]),2)) < latMin:
latMin = abs(round(float(vals[0]),2))
#determines the max and min longitudes
if longMax == None:
longMax = abs(round(float(vals[1]),2))
longMin = abs(round(float(vals[1]),2))
elif abs(round(float(vals[1]),2)) > longMax:
longMax = abs(round(float(vals[1]),2))
elif abs(round(float(vals[1]),2)) < longMin:
longMin = abs(round(float(vals[1]),2))
#determine the max altitude
if altMax == None:
altMax = abs(float(vals[2]))
elif abs(float(vals[2])) > altMax:
altMax = abs(float(vals[2]))
pbar.update() #updates the prgress bar
file.close() #closes the file
file = open(fileName)
interval = 5 - int(1/interval)
if interval == 0:
interval = 1
matrix = makeMatrix(int(((round((float(latMax)-float(latMin)),2) * 100))/interval)+1,int(((round((float(longMax)-float(longMin)),2) * 100))/interval)+1)
pbar = ProgressBar(size,"Filling matrix...")
count = 0
num = 0
for line in file:
points = []
vals = line.rstrip().split(',')
#for element in vals:
#temp = element.split(' ')
#points.append(temp[len(temp)-1])
#vals = points
x = int((round((abs(float(vals[0])) - float(latMin)),2) * 100))
y = int((round((abs(float(vals[1])) - float(longMin)),2) * 100))
try:
matrix[int(x/interval)][int(y/interval)] = altMax - abs(float(vals[2]))
num += 1
except:
count = count + 1
pbar.update()
file.close()
matrix = findAllMissing(matrix,ACCURACY)
writeBack(matrix)
print()
print("Number missed...................= ",count)
print("Number added....................= ",num)
print("Number of alts..................= ",size)
print("Size of the matrix..............= ",(len(matrix) * len(matrix[0])))
print("Matrix height...................= ",len(matrix) - 1)
print("Matrix width....................= ",len(matrix[0]) - 1)
print("Difference in latitude..........= ",int(round((float(latMax)-float(latMin)),2) * 100))
print("Difference in longitude.........= ",int(round(float((longMax)-float(longMin)),2) * 100))
print("Average dist between points.....= ", interval," cm(s)")
print("Average number of Longs per alt.= ",round(size/len(longs),2))
print(latMin,"--->",latMax)
print(longMin,"--->",longMax)
a = input("**Hit ENTER to close**") #keeps the terminal open
return
def writeBack(matrix):
size = len(matrix) + len(matrix[0]) + int((len(matrix) * len(matrix[0])))
pbar = ProgressBar(size,"Writing points to file...") #initializes the progress bar
line = ""
i = 0
#gets the appropriate file name
it = str(getIteration())
newFile = "./3D Maps/3D Latitudes(" + it + ").txt"
#opens the file
file = open(newFile,"w+")
#writes the latitudes to the text file
while i < len(matrix):
if i < len(matrix) - 1:
line = line + str(i) + ","
else:
line = line + str(i) + "\n"
pbar.update()
i = i + 1
file.write(line)
file.close
newFile = "./3D Maps/3D Longitudes(" + it + ").txt"
#opens the file
file = open(newFile,"w+")
line = "" #resets line
i = 0 #resets count
#writes the longitudes to the text file
while i < len(matrix[0]):
if i < len(matrix[0]) - 1:
line = line + str(i) + ","
else:
line = line + str(i) + "\n"
pbar.update()
i = i + 1
file.write(line)
file.close
newFile = "./3D Maps/3D Matrix(" + it + ").txt"
i = 0
#opens the file
file = open(newFile,"w+")
#writes the matrix to the textfile
for row in matrix:
line = "" #resets line
count = 0 #resets count
for element in row:
if count < len(matrix[0]) - 1:
line = line + str(element) + ","
else:
line = line + str(element) + '\n'
pbar.update()
count = count + 1
file.write(line)
file.close()
temp = "Done. 3D Map("+it+") has been created. You may now close the program."
print(temp)
return
def main():
"""
read from syslog file into XALT db.
"""
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"' + v + '"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(args.confFn)
syslogFile = args.syslog
icnt = 0
t1 = time.time()
try:
rmapT = Rmap(args.rmapD).reverseMapT()
except Exception as e:
print(e, file=sys.stderr)
print("Failed to read reverseMap file -> exiting")
sys.exit(1)
lnkCnt = 0
pkgCnt = 0
runCnt = 0
badCnt = 0
count = 0
recordT = {}
fnA = [args.leftover, syslogFile]
parseSyslog = ParseSyslog(args.leftover)
#-----------------------------
# Figure out size in bytes.
fnSz = 0
for fn in fnA:
if (not os.path.isfile(fn)):
continue
fnSz += os.path.getsize(fn)
#----------------------------------------------------------
# Count the number and sum the run_time for all scalar jobs
filter = Filter(100)
pbar = ProgressBar(maxVal=fnSz, fd=sys.stdout)
for fn in fnA:
if (not os.path.isfile(fn)):
continue
old = (fn == args.leftover)
lineNo = 0
f = open(fn, 'r')
for line in f:
lineNo += 1
count += len(line)
pbar.update(count)
if (not ("XALT_LOGGING" in line)):
continue
try:
t, done = parseSyslog.parse(line, args.syshost, old)
except Exception as e:
#print(e, file=sys.stderr)
#print("lineNo:",lineNo,"file:",fn,"line:",line, file=sys.stderr)
#print("Now continuing processing!", file=sys.stderr)
continue
if (not done or t['kind'] != "run"):
continue
##################################
# If the json conversion fails,
# then ignore record and keep going
value = False
try:
value = json.loads(t['value'])
filter.register(value)
except Exception as e:
#print("fn:",fn,"line:",lineNo,"value:",t['value'],file=sys.stderr)
continue
f.close()
pbar.fini()
filter.report_stats()
badsyslog = 0
count = 0
parseSyslog = ParseSyslog(args.leftover)
pbar = ProgressBar(maxVal=max(fnSz, 1), fd=sys.stdout)
for fn in fnA:
if (not os.path.isfile(fn)):
continue
old = (fn == args.leftover)
f = open(fn, 'r')
for line in f:
count += len(line)
pbar.update(count)
if (not ("XALT_LOGGING" in line)):
continue
try:
t, done = parseSyslog.parse(line, args.syshost, old)
except Exception as e:
badsyslog += 1
continue
if (not done):
continue
##################################
# If the json conversion fails,
# then ignore record and keep going
try:
value = json.loads(t['value'])
except Exception as e:
continue
try:
XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " +
t['syshost'])
if (t['kind'] == "link"):
XALT_Stack.push("link_to_db()")
xalt.link_to_db(rmapT, value)
XALT_Stack.pop()
lnkCnt += 1
elif (t['kind'] == "run"):
if (filter.apply(value)):
XALT_Stack.push("run_to_db()")
xalt.run_to_db(rmapT, value)
XALT_Stack.pop()
runCnt += 1
elif (t['kind'] == "pkg"):
XALT_Stack.push("pkg_to_db()")
xalt.pkg_to_db(t['syshost'], value)
XALT_Stack.pop()
pkgCnt += 1
else:
print("Error in xalt_syslog_to_db", file=sys.stderr)
XALT_Stack.pop()
except Exception as e:
print(e, file=sys.stderr)
badCnt += 1
f.close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ",
runCnt, ", pkgCnt: ", pkgCnt, ", badCnt: ", badCnt, ", badsyslog: ",
badsyslog)
# if there is anything left in recordT file write it out to the leftover file.
parseSyslog.writeRecordT()
def main():
"""
read from syslog file into XALT db.
"""
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"'+v+'"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(dbConfigFn(args.dbname))
syslogFile = args.syslog
# should add a check if file exists
num = int(capture("cat "+syslogFile+" | wc -l"))
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
lnkCnt = 0
runCnt = 0
badCnt = 0
count = 0
recordT = {}
if (num == 0):
return
pbar = ProgressBar(maxVal=num)
fnA = [ args.leftover, syslogFile ]
for fn in fnA:
if (not os.path.isfile(fn)):
continue
f=open(fn, 'r')
for line in f:
if (not ("XALT_LOGGING" in line)):
continue
t, done = parseSyslog(line, recordT)
if (not done):
continue
try:
XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " + t['syshost'])
if ( t['kind'] == "link" ):
XALT_Stack.push("link_to_db()")
xalt.link_to_db(rmapT, json.loads(t['value']))
XALT_Stack.pop()
lnkCnt += 1
elif ( t['kind'] == "run" ):
XALT_Stack.push("run_to_db()")
xalt.run_to_db(rmapT, json.loads(t['value']))
XALT_Stack.pop()
runCnt += 1
else:
print("Error in xalt_syslog_to_db", file=sys.stderr)
XALT_Stack.pop()
except Exception as e:
print(e, file=sys.stderr)
badCnt += 1
count += 1
pbar.update(count)
f.close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ", runCnt, ", badCnt: ", badCnt)
leftover = args.leftover
if (os.path.isfile(leftover)):
os.rename(leftover, leftover + ".old")
# if there is anything left in recordT file write it out to the leftover file.
if (recordT):
f = open(leftover, "w")
for key in recordT:
r = recordT[key]
s = r.prt("XALT_LOGGING V=2", key)
f.write(s)
f.close()
def main():
"""
read from syslog file into XALT db.
"""
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"' + v + '"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(dbConfigFn(args.dbname))
syslogFile = args.syslog
# should add a check if file exists
num = int(capture("cat " + syslogFile + " | wc -l"))
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
lnkCnt = 0
runCnt = 0
badCnt = 0
count = 0
recordT = {}
if (num == 0):
return
pbar = ProgressBar(maxVal=num)
fnA = [args.leftover, syslogFile]
for fn in fnA:
if (not os.path.isfile(fn)):
continue
f = open(fn, 'r')
for line in f:
if (not ("XALT_LOGGING" in line)):
continue
t, done = parseSyslog(line, recordT)
if (not done):
continue
try:
XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " +
t['syshost'])
if (t['kind'] == "link"):
XALT_Stack.push("link_to_db()")
xalt.link_to_db(rmapT, json.loads(t['value']))
XALT_Stack.pop()
lnkCnt += 1
elif (t['kind'] == "run"):
XALT_Stack.push("run_to_db()")
xalt.run_to_db(rmapT, json.loads(t['value']))
XALT_Stack.pop()
runCnt += 1
else:
print("Error in xalt_syslog_to_db", file=sys.stderr)
XALT_Stack.pop()
except Exception as e:
print(e, file=sys.stderr)
badCnt += 1
count += 1
pbar.update(count)
f.close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ",
runCnt, ", badCnt: ", badCnt)
leftover = args.leftover
if (os.path.isfile(leftover)):
os.rename(leftover, leftover + ".old")
# if there is anything left in recordT file write it out to the leftover file.
if (recordT):
f = open(leftover, "w")
for key in recordT:
r = recordT[key]
s = r.prt("XALT_LOGGING V=2", key)
f.write(s)
f.close()
def main():
"""
Walks the list of users via the passwd_generator and load the
link and run files.
"""
# Find transmission style
transmission = os.environ.get("XALT_TRANSMISSION_STYLE")
if (not transmission):
transmission = "@XALT_TRANSMISSION_STYLE@"
if (not transmission):
transmission = "file"
transmission = transmission.lower()
# Push command line on to XALT_Stack
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"'+v+'"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(args.confFn)
num = int(capture("getent passwd | wc -l"))
pbar = ProgressBar(maxVal=num)
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
iuser = 0
lnkCnt = 0
runCnt = 0
pkgCnt = 0
for user, hdir in passwd_generator():
XALT_Stack.push("User: "******"link")
if (os.path.isdir(xaltDir)):
iuser += 1
linkFnA = files_in_tree(xaltDir, "*/link." + args.syshost + ".*.json")
XALT_Stack.push("link_json_to_db()")
lnkCnt += link_json_to_db(xalt, args.listFn, rmapT, args.delete, linkFnA)
XALT_Stack.pop()
xaltDir = build_xaltDir(user, hdir, transmission, "run")
if (os.path.isdir(xaltDir)):
runFnA = files_in_tree(xaltDir, "*/run." + args.syshost + ".*.json")
XALT_Stack.push("run_json_to_db()")
runCnt += run_json_to_db(xalt, args.listFn, rmapT, args.delete, runFnA)
XALT_Stack.pop()
xaltDir = build_xaltDir(user, hdir, transmission, "pkg")
if (os.path.isdir(xaltDir)):
pkgFnA = files_in_tree(xaltDir, "*/pkg." + args.syshost + ".*.json")
XALT_Stack.push("pkg_json_to_db()")
pkgCnt += pkg_json_to_db(xalt, args.listFn, args.syshost, args.delete, pkgFnA)
XALT_Stack.pop()
icnt += 1
v = XALT_Stack.pop()
carp("User",v)
pbar.update(icnt)
xalt.connect().close()
pbar.fini()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
print("num users: ", iuser, ", num links: ", lnkCnt, ", num pkgs: ", pkgCnt, ", num runs: ", runCnt)
def main():
"""
read from syslog file into XALT db.
"""
sA = []
sA.append("CommandLine:")
for v in sys.argv:
sA.append('"' + v + '"')
XALT_Stack.push(" ".join(sA))
args = CmdLineOptions().execute()
xalt = XALTdb(ConfigFn)
syslogFile = args.syslog
# should add a check if file exists
num = int(capture("cat " + syslogFile + " | wc -l"))
icnt = 0
t1 = time.time()
rmapT = Rmap(args.rmapD).reverseMapT()
lnkCnt = 0
runCnt = 0
badCnt = 0
count = 0
if num != 0:
pbar = ProgressBar(maxVal=num)
if (os.path.isfile(syslogFile)):
f = open(syslogFile, 'r')
for line in f:
if "XALT_LOGGING" in line:
i = line.find("link:")
if i == -1:
i = line.find("run:")
if i == -1:
continue # did not find a link or run, continue to next line
array = line[i:].split(":")
type = array[0].strip()
syshost = array[1].strip()
try:
resultT = json.loads(base64.b64decode(array[2]))
XALT_Stack.push("XALT_LOGGING: " + type + " " +
syshost)
# XALT_Stack.push("XALT_LOGGING resultT: " + resultT)
if (type == "link"):
XALT_Stack.push("link_to_db()")
xalt.link_to_db(rmapT, resultT)
XALT_Stack.pop()
lnkCnt += 1
elif (type == "run"):
XALT_Stack.push("run_to_db()")
xalt.run_to_db(rmapT, resultT)
XALT_Stack.pop()
runCnt += 1
else:
print("Error in xalt_syslog_to_db")
XALT_Stack.pop()
except:
badCnt += 1
# figure out length of array[2], as it might be
# near the syslog message size limit
strg = array[2]
lens = len(strg)
# lenx = lens - (lens % 4 if lens % 4 else 4)
print(
"xalt_syslog_to_db: undecodable entry! length: ",
lens)
# resultT = base64.decodestring(strg[:lenx])
# print("result: ",result)
count += 1
pbar.update(count)
pbar.fini()
# what should be done if there are errors?
# what went wrong?
# how do we restart?
#
# xalt.connect().close()
t2 = time.time()
rt = t2 - t1
if (args.timer):
print("Time: ", time.strftime("%T", time.gmtime(rt)))
#
print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ",
runCnt, ", badCnt: ", badCnt)
