def main(*arguements):
# --------------------------------------------------------
# Parse command line, to fetch session input file
# --------------------------------------------------------
if len(sys.argv) >= 2:
ripFilePath = sys.argv[1]
# If user did not specify .rip file
else:
print "User must specify input file name (e.g., at command line type GARM.py user_input.rip)."
sys.exit(-1)
# If .ip file does not exist
if not os.path.exists(ripFilePath):
print ("Cannot find or open runtime inputs file(%s)" % (ripFilePath))
sys.exit(-1)
dir = "tree100perlandscape/"
# bird_types = ['St','Mu','Gl','Cr','Ax',Sh,Tu
# Skipped Du, Re, Ru
bird_types = ["Ha"]
file_types = ["_tree_100_1_10.asc", "_tree_100_1_100.asc"]
# file_types = ['_tree_100_1_2.asc','_tree_100_1_5.asc','_tree_100_1_10.asc','_tree_100_1_100.asc']
grid_file_out = "aus_bird_unicor.rip"
for bird in bird_types:
for type in file_types:
print "now running " + bird + type
header_dict, data = FileIO.loadFile(ripFilePath, header_lines=16)
header_dict["XY_Filename"] = str(bird + "_XY.csv")
header_dict["Grid_Filename"] = str(dir + bird + type)
header_dict["Session_label"] = bird + "_type"
FileIO.outputGrid(grid_file_out, ["null", "null"], header_dict=header_dict)
cd_matrix = RunUNICOR(grid_file_out)
def FileIO_test():
data ="1\n2\n3\n4\n"
rt = FileIO.write('WRITE_TEST', data, 'w')
assert(rt == 0)
c_list = []
(rt, c_list) = File.read("WRITE_TEST", 3)
assert(rt == 0)
assert(len(c_list) == 3)
# 0 이면 all line -> list 로
(rt, c_list) = FileIO.read("WRITE_TEST", 0)
assert(rt == 0)
assert(len(c_list) == 4)
list = []
list.append('AA')
list.append('BB')
rt = FileIO.write('TEST01', list)
assert(rt == 0)
(rt, c_list) = FileIO.read("TEST01", 2)
assert(rt == 0)
assert(len(c_list) == 2)
return
def main(trainMode, binarized, testDir, resultFile):
dir1 = r'R:\masters\fall2013\COMP579A\project\aclImdb\train\pos';
dir2 = r'R:\masters\fall2013\COMP579A\project\aclImdb\train\neg';
corpusName = r'corpus\binCorpus.txt';
if(not binarized):
corpusName = r'corpus\corpus.txt';
if(trainMode):
if(binarized):
buildBinarizedCorpus(dir1, corpusName);
buildBinarizedCorpus(dir2, corpusName);
else:
buildCorpus(dir1, corpusName);
buildCorpus(dir2, corpusName);
dictionary = loadCorpus(corpusName);
testText = '';
print('test for dir' , testDir);
for dir_entry in os.listdir(testDir):
if binarized:
testText = getBinTestText(os.path.join(testDir, dir_entry));
else:
testText = getTestText(os.path.join(testDir, dir_entry));
result =test(testText, dictionary);
FileIO.wrtieToFile(resultFile, 'a' ,"The file " + os.path.join(testDir, dir_entry) + " is classified as " + result + "\n");
print("Done....");
def login(home_path, today, file_write = 0):
try :
(ok, login_l, other_l) = login_list(os.path.join(home_path,'.ac_info'))
if not ok : return (0, home_path + ' ' + l)
#(1, [('20051010', '20051010'), ('20051012', '20051015')], [])
(ok, r) = _login(login_l, today)
if not ok :
return (0, r)
login_l2 = login_format(r)
reply = join_list(login_l2, other_l)
i = len(login_l2) - len(login_l)
log = '%s %d row added' % ( os.path.join(home_path,'.ac_info'), i)
if( not os.path.exists(home_path)) :
return (0, reply, home_path + ' not exists')
if (file_write) :
FileIO.write(os.path.join(home_path,'.ac_info'), reply)
return (1, reply, log)
except :
t, v, tb = sys.exc_info()
print 'login exception... (%s:%s)' % (t, v)
return (0, log , log)
def writeCustomToFile(customPrograms, customName):
'''Store custom application information'''
tempDir = tempfile.gettempdir()
tempLocation = os.path.join(tempDir,'AssetManagerTemp')
tempLocation = os.path.join(tempLocation,'CustomMenuItems.ini')
data = '#Custom Application:'
section = ['CustomPrograms', 'CustomName']
key = ['Programs', '\n', 'Names']
value = []
for item in customPrograms:
value.append(item)
value.append('\n')
for item in customName:
value.append(item)
data = FileIO.build(data, section, key, value)
FileIO.write(data, tempLocation)
def ktf_notify(id) : try : ok , path = home_fullpath(id) if not os.path.exists(os.path.join(path, '.ktf_notify')) : return (0, '','',[]) ok, lines = FileIO.read(os.path.join(path, '.ktf_notify')) if not ok : return (0, '','',[]) hash = TokenList.decode(lines[0].rstrip()) email_list = [] for line in lines[1:] : email_list.append(line.rstrip()) return (1, hash['level'], hash['type'] , email_list) except : t, v, tb = sys.exc_info() log = 'ktf_notify(%s:%s)' % (t, v) print log return (0, '','',[])
def login_list(file) :
try :
(ok, lines) = FileIO.read(file)
reply = []
other_l = []
for line in lines :
try :
#NOT FOUND
if line.find('LOGIN\t') == -1 :
other_l.append(line.rstrip())
continue
# LOGIN \t A:B
(dummy, login_date) = line.rstrip().split('\t')
(sdate, edate) = login_date.split(':')
reply.append((sdate, edate))
except :
pass
return (1, reply, other_l)
except :
t, v, tb = sys.exc_info()
log = 'login list ex... (%s:%s)' % (t, v)
return (0, log ,log)
def test(testText, dictionary):
classCntTxt = FileIO.readFile('corpus\classCount.txt');
classCnt = {};
for line in classCntTxt.split('\n'):
statistics = line.split('\t');
if len(statistics) > 1:
classCnt[statistics[0]] = statistics[1];
return Classifier.naiveBayes(dictionary, classCnt, testText);
def __init__(self, busName, fileBufferSize, bitResolution, upLimit=None, loLimit=None, p2pShrink=None):
self.busName = busName
self.fileBufferSize = fileBufferSize
self.bitResolution = bitResolution
self.upLimit = upLimit
self.loLimit = loLimit
self.p2pShrink = p2pShrink
self.fileIO = FileIO()
def modifyTrainTxt():
mapp = {
'75': '1267',
'13': '1015',
'24': '1261',
'21': '1031',
'64': '1010',
'0': '1006'
}
data_types = ['data_p', 'data_p_c']
for id in mapp:
for dt in data_types:
FileIO.modify_train(
'modified_training_files/' + dt + '/train.txt',
'modified_training_files/' + dt + '/train_' + id + '.txt', id)
FileIO.modify_train(
'modified_training_files/' + dt + '/val.txt',
'modified_training_files/' + dt + '/val_' + id + '.txt', id)
def read_n_images(self):
nega_file = '../data/ms_negative.csv'
lines = FileIO.csv_reader(nega_file)
n_imgs = []
for line in lines:
task_x = line['task_x']
task_y = line['task_y']
n_imgs.append([int(task_x), int(task_y)])
return n_imgs
def saveCenters(doOnlyFulls = False, trainingFolder = '../data/nicicon/csv/train/csv/' , suffix = '_train.csv', savePath = '../data/newMethodTraining/allCentersNic.csv'):
"""This file computes representatives of classes in the nicicon database and saves it to the ./data/csv/allCentersNic.csv"""
files = ['accident','bomb','car','casualty','electricity','fire','firebrigade','flood','gas','injury','paramedics', 'person', 'police','roadblock']
centerSet = []
my_names = []
my_isFull = []
f = FileIO()
extr_train = Extractor(trainingFolder)
train_features, \
train_isFull, \
train_classId, \
train_names = extr_train.loadniciconfolders()
for mfile in files[:]:
#Get every instance in the given class
names, isFull, features = f.load(trainingFolder + mfile + '/' + mfile + suffix)
#Initialise centers as 0
nowCenter = np.zeros(len(features[0]))
if not doOnlyFulls:
#Use every full and partial instance in the class and sum them
totalNumOfInstances = len(features)
for instance in features:
nowCenter += instance
else:
#Use only full instances in the class and sum them
totalNumOfInstances = 0
for i in range(len(features)):
if isFull[i] == True:#Check the prints to see if this works fine (See if the totalNumOfInstances is meaningful
nowCenter+=features[i]
totalNumOfInstances+=1
print mfile,'Used ', totalNumOfInstances, 'of instances to find the class center'
nowCenter = nowCenter/totalNumOfInstances
centerSet.append(nowCenter)#Add the mean feature vector for this representative
my_names.append(mfile)#The name of the representative
my_isFull.append(0)#Just set anything, not necessary for class representative
f.save(my_isFull,my_names,centerSet,savePath)
def getBinTestText(fileName) :
fileText = FileIO.readFile(fileName);
tokens = Tokenizer.tokenizer(fileText);
wordOccurence = {};
resultTokens = "";
for token in tokens.split("\n"):
if token not in wordOccurence:
wordOccurence[token] = 1;
resultTokens = resultTokens + token + "\n";
return resultTokens;
def readCSV_CSTS(self,fileName,delimiter=","):
"""Reads csv file with CSTS variable
fileName = str() path/filename of csv file
checks new vars into ProjectMaker
returns variable name
"""
csts = FIO.csvReaderCSTS(fileName,delimiter=delimiter)
self.createVariable(csts.name, "CSTS", csts.newVars[csts.name])
return csts.name
def writePRJ(self):
if len(self.matrices.keys()) != 0:
self.processMatrices()
self.prjInfo['data'] = self.projPrefix
if self.arc == 1:
self.prjInfo['gis'] = self.projPrefix
self.prjInfo['projection'] = self.map.projectionName
self.setScreenInfo()
if self.prj == 1:
gisFile = OS.path.join(self.originalDir,self.gisFileMain+".gis")
if self.originalPrefix != self.projPrefix:
newGIS = OS.path.join(self.projectDir,self.projPrefix+".gis")
self.copyFile2File(gisFile,newGIS)
else:
gisFile = OS.path.join(self.originalDir,self.gisFileMain+".gis")
self.copyFile2Dir(gisFile,self.projectDir)
self.prjInfo['gis'] = self.projPrefix
self.prjInfo['main'] = self.createOutputFile(self.projPrefix, ".prj")
FIO.projWriter(self.prjInfo)
def loadCorpus(corpusName):
dictionary = {};
fileText = FileIO.readFile(corpusName);
for line in fileText.split('\n'):
words = line.split('\t');
if len(words) > 2:
if words[0] not in dictionary:
dictionary[words[0]] = {};
dictionary[words[0]][words[2]] = words[1];
return dictionary;
def readAttributeInfo(self):
"""Reads CSV File for Project of type CS
self.initial = (dict) with first row names as keys and list of values
as values
"""
file = self.fileName
info = FIO.csvReaderCS(file)
self.initial = info.newVars
self.addInitialShapesVariable()
def readCSV_CSTS(self, fileName, delimiter=","):
"""Reads csv file with CSTS variable
fileName = str() path/filename of csv file
checks new vars into ProjectMaker
returns variable name
"""
csts = FIO.csvReaderCSTS(fileName, delimiter=delimiter)
self.createVariable(csts.name, "CSTS", csts.newVars[csts.name])
return csts.name
def readAttributeInfo(self):
"""Reads CSV File for Project of type CS
self.initial = (dict) with first row names as keys and list of values
as values
"""
file = self.fileName
info = FIO.csvReaderCS(file)
self.initial = info.newVars
self.addInitialShapesVariable()
def testImages(): filename = getTemporaryFilename( ext='png' ) image = FileIO.readImage( getThisDirectory() + '\\' + 'peppers.jpg' ) assert image is not None FileIO.writeImage( filename, image ) image_copy = FileIO.readImage( filename ) assert image == image_copy """ app = QApplication( sys.argv ) display = ImageDisplay( image ) sys.exit( app.exec_() ) """ return True
def output_cdmatrix(logfHndl,cd_matrix, output_filename,data_dir,stringpts): ''' output_cdmatrix() This writes the cost distance matrix to file. Checks for duplicate points ''' # Timing events: start start_time1 = datetime.datetime.now() # Create a full matrix, recall that cdmatrix comes in as a lower tri-diagonal temp = np.transpose(cd_matrix) cd_matrix = temp + cd_matrix # Check for duplicates and copy lines uni_strpts = count_unique(stringpts) duplicates_strpts = np.where(uni_strpts[1]>1)[0] if len(duplicates_strpts) != 0: # Loop through duplicates for i in xrange(len(duplicates_strpts)): # Get dupliccate value strdouble = uni_strpts[0][duplicates_strpts[i]] # Find location in strpts strpts_loc = np.where(strdouble == np.asarray(stringpts))[0] # Then copy the last duplicate (always the last) into the other lines copyrow = cd_matrix[strpts_loc[-1],:] copycol = cd_matrix[:,strpts_loc[-1]] for jspot in strpts_loc: cd_matrix[jspot,:] = copyrow cd_matrix[:,jspot] = copycol # Then make sure that all dubplicate spots to cuplicate spots are 0 for kspot in strpts_loc: cd_matrix[jspot][kspot] = 0. # Make sure diagnols are 0 cd_matrix[np.diag_indices_from(cd_matrix)] = 0. FileIO.outputGrid(data_dir+output_filename,cd_matrix,delimiter=',') stringout = 'The file '+str(data_dir+output_filename)+' has been created in: '+\ str(datetime.datetime.now() -start_time1) FileIO.logMsg(logfHndl,stringout+'\n')
def readCSV_TS(self,fileName,delimiter=","):
"""Reads csv file with TS variables
fileName = str() path/filename of csv file
checks new vars into ProjectMaker
returns list of new variable names
"""
ts = FIO.csvReaderTS(fileName,delimiter=delimiter)
for i in ts.names:
self.createVariable(i, "TS", [ts.newVars[i]])
return ts.names
def readJoinCSV(self,fileName,delimiter=","):
"""Reads csv file with CS variables
fileName = str() path/filename of csv file
nameField = str() string to identify which column will be used to
match with csnames
checks new vars into ProjectMaker
returns list of new variable names
"""
self.data2Join = FIO.csvReaderCS(fileName,delimiter=delimiter)
self.joinData = self.createJoinTable()
def openFile(self, event):
wildCard = "CSV or TXT Files Containing Coordinate Data (*)|*"
fileDialog = wx.FileDialog(self,
"Open NMR or FID Data",
wildcard=wildCard,
style=wx.FD_OPEN | wx.FD_FILE_MUST_EXIST)
if fileDialog.ShowModal() == wx.ID_OK:
self.filePath = fileDialog.GetPath()
plotData = FileIO.openFile(self.filePath)
self.graphPanel.newDataSet(plotData[0], plotData[1])
fileDialog.Destroy()
def readCSV_TS(self, fileName, delimiter=","):
"""Reads csv file with TS variables
fileName = str() path/filename of csv file
checks new vars into ProjectMaker
returns list of new variable names
"""
ts = FIO.csvReaderTS(fileName, delimiter=delimiter)
for i in ts.names:
self.createVariable(i, "TS", [ts.newVars[i]])
return ts.names
def GPX_positive(self):
gpx_file = '../data/gpx_nodes.csv'
lines = FileIO.csv_reader(gpx_file)
p_imgs_raw = []
for line in lines:
task_x = line['task_x']
task_y = line['task_y']
img = '%s-%s.jpeg' % (task_x, task_y)
p_imgs_raw.append(img)
p_imgs = list(set(p_imgs_raw))
return p_imgs
def __init__(self, settings_loc):
self.settings = FileIO.get_settings(settings_loc)
self.sensors_loc = os.path.relpath(self.settings["sensors_location"])
self.results_loc = os.path.relpath(self.settings["results"])
self.experiments = None
self.current_plot = PlotObserver()
self.options = [
'get default frequency', 'get sensor data', 'get sensors',
'print results', 'activate', 'add description', 'add sensor',
'stop', 'exit'
]
def OSM_positive(self):
osm_file = '../data/guinea_highway.csv'
lines = FileIO.csv_reader(osm_file)
p_imgs_raw = []
for line in lines:
task_x = line['task_x']
task_y = line['task_y']
img = '%s-%s.jpeg' % (task_x, task_y)
p_imgs_raw.append(img)
p_imgs = list(set(p_imgs_raw))
return p_imgs
def Part2(createData):
# optical flow and motion direction histogram calculation
v = 100
# mdh_all = OM.createMotionDirectionHistograms('Oberstdorf16-shots.csv', 'videos/oberstdorf16.mp4', v, False, True)
# FileIO.save_histograms_to_file('mdh_16_' + str(v) + '.csv', mdh_all)
if createData:
SVM.save_shot_images('videos/oberstdorf16.mp4', SVM.SSI_CENTER, 'Oberstdorf16-shots.csv', False)
#svm training and predicting
mdh_training = FileIO.read_histograms_from_file('mdh_8_' + str(v) + '.csv')
mdh_test = FileIO.read_histograms_from_file('mdh_16_' + str(v) + '.csv')
predicted_labels = SVM.svm_use(mdh_training, mdh_test)
stitched_shots, all_shots, outstitched_shots = SVM.get_results(
predicted_labels, 'Oberstdorf16-shots.csv', True)
return stitched_shots, all_shots, outstitched_shots
def readJoinCSV(self, fileName, delimiter=","):
"""Reads csv file with CS variables
fileName = str() path/filename of csv file
nameField = str() string to identify which column will be used to
match with csnames
checks new vars into ProjectMaker
returns list of new variable names
"""
self.data2Join = FIO.csvReaderCS(fileName, delimiter=delimiter)
self.joinData = self.createJoinTable()
def load_fasta_background(filename, center='.'):
fasta_list = FileIO.read_fasta(filename)
result = fasta_to_chunks(fasta_list)
if (center != '.'):
result = filter_chunks(result, center)
result = result[0:300000]
#Concatenate because I don't have enough ram
formA = DataManipulation.list_to_formA(result)
return dict([('seq', result), ('formA', formA)])
def load_fasta_background(filename, center='.'):
fasta_list = FileIO.read_fasta(filename);
result = fasta_to_chunks(fasta_list);
if(center != '.'):
result = filter_chunks(result, center);
result = result[0:300000]; #Concatenate because I don't have enough ram
formA = DataManipulation.list_to_formA(result);
return dict([('seq',result),
('formA',formA)]);
def take_frame_from_end(video_info, video_path, output_folder):
frame_filename = os.path.join(output_folder, FileIO.get_frame_filename(video_path, "end", config.video_frames_extension))
end_location = str((float(video_info[2]) - float(video_info[1])))
if not end_location.startswith("10."):
end_location = "0" + end_location
ffmpeg_string = 'ffmpeg -i {} -ss 00:00:{} -vframes 1 "{}"'.format(video_path, end_location, frame_filename)
os.system(ffmpeg_string)
return frame_filename
def read_p_images(self):
osm_file = '../data/guinea_highway.csv'
lines = FileIO.csv_reader(osm_file)
p_imgs_raw = []
for line in lines:
task_x = line['task_x']
task_y = line['task_y']
p_imgs_raw.append([int(task_x), int(task_y)])
p_imgs = [
list(t) for t in set(tuple(element) for element in p_imgs_raw)
] # remove duplicate
return p_imgs
def read_p_images(self):
gpx_file = '../data/gpx_nodes.csv'
lines = FileIO.csv_reader(gpx_file)
p_imgs_raw = []
for line in lines:
task_x = line['task_x']
task_y = line['task_y']
p_imgs_raw.append([int(task_x), int(task_y)])
p_imgs = [
list(t) for t in set(tuple(element) for element in p_imgs_raw)
]
return p_imgs
def writeDHT(self, delimiter=","):
"""Writes the STARS header file.
writes projectName.dat file for STARS
"""
tInfo = [[self.timeType, self.stringTime[0], self.stringTime[-1]]]
csnames = self.varDict["csnames"]
name = [["csnames"]]
vals = UTIL.splitList(csnames.values[0])
info = tInfo + name + vals
dhtFile = self.createOutputFile(self.projPrefix, ".dht")
fdht = FIO.csvWriter(dhtFile, info, delimiter=" ")
def writePRJ(self):
if len(self.matrices.keys()) != 0:
self.processMatrices()
self.prjInfo['data'] = self.projPrefix
if self.arc == 1:
self.prjInfo['gis'] = self.projPrefix
self.prjInfo['projection'] = self.map.projectionName
self.setScreenInfo()
if self.prj == 1:
gisFile = OS.path.join(self.originalDir, self.gisFileMain + ".gis")
if self.originalPrefix != self.projPrefix:
newGIS = OS.path.join(self.projectDir,
self.projPrefix + ".gis")
self.copyFile2File(gisFile, newGIS)
else:
gisFile = OS.path.join(self.originalDir,
self.gisFileMain + ".gis")
self.copyFile2Dir(gisFile, self.projectDir)
self.prjInfo['gis'] = self.projPrefix
self.prjInfo['main'] = self.createOutputFile(self.projPrefix, ".prj")
FIO.projWriter(self.prjInfo)
def writeDHT(self,delimiter=","):
"""Writes the STARS header file.
writes projectName.dat file for STARS
"""
tInfo = [[self.timeType,self.stringTime[0],self.stringTime[-1]]]
csnames = self.varDict["csnames"]
name = [["csnames"]]
vals = UTIL.splitList(csnames.values[0])
info = tInfo + name + vals
dhtFile = self.createOutputFile(self.projPrefix, ".dht")
fdht = FIO.csvWriter(dhtFile, info, delimiter=" ")
def output_paths(logfHndl,paths,paths_file_name,data_dir):
'''
output_paths() - output the full list of paths between points,
and shortest path distance for a given path
'''
# Timing events: start
start_time1 = datetime.datetime.now()
# Open file to write to
fout = open(data_dir+paths_file_name, 'w')
# Write out title information
fout.write('Source,Destination,PathLength,PathConnections\n')
# Begin loop through point combinations
for ipath in xrange(len(paths)):
# Print out path source, destination, length
fout.write(paths[ipath][0])
fout.write(',')
fout.write(paths[ipath][1])
fout.write(',')
fout.write(str(paths[ipath][2]))
fout.write(',')
# Print out the path connections
if paths[ipath][3] != None:
for ipathlen in xrange(len(paths[ipath][3])):
fout.write(paths[ipath][3][ipathlen])
fout.write(',')
else:
fout.write('nan')
fout.write('\n')
# Logging information
stringout = 'The file '+str(data_dir+paths_file_name)+' has been created in: '+\
str(datetime.datetime.now() -start_time1)
FileIO.logMsg(logfHndl,stringout+'\n')
# Close file
fout.close
def gen(train_x, train_y, test_x, test_y, parameters, length, cutoff):
# Variable Declarations
step_count = length
threshold = cutoff
z8 = np.empty((step_count + 1, )) # quintrophic omnivory
y8 = np.empty((step_count + 1, ))
x8 = np.empty((step_count + 1, ))
c8_1 = np.empty((step_count + 1, ))
c8_2 = np.empty((step_count + 1, ))
p8_1 = np.empty((step_count + 1, ))
p8_2 = np.empty((step_count + 1, ))
p8_3 = np.empty((step_count + 1, ))
e = 0.000000001
dt = 0.01
d_c1 = 0.25
d_c2 = 0.25
d_x = 0.1075
d_z = 0.06
min_y, max_y = (0.007, 0.013)
grid_fineness = 500
count = 0
# Initial Conditions for the food web topologies labeled 0 through 8. All start at 0.5
z8[0], y8[0], x8[0], c8_1[0], c8_2[0], p8_1[0], p8_2[0], p8_3[0] = (
0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5) # quintrophic omnivory
# Iterate through mortality rates
for d_y in np.linspace(min_y, max_y, grid_fineness):
print d_z, ",", d_y, ",", d_x, ",", d_c1, ",", d_c2
# Evaulate time series
for i in range(step_count):
z8[i + 1], y8[i + 1], x8[i + 1], c8_1[i + 1], c8_2[i + 1], p8_1[
i + 1], p8_2[i + 1], p8_3[i + 1] = cm.quintrophicOmnivory(
z8[i], y8[i], x8[i], c8_1[i], c8_2[i], p8_1[i], p8_2[i],
p8_3[i], d_z, d_y, d_x, d_c1, d_c2, dt)
# Check for coexistence and record the data if the species coexist
if (z8[step_count] > e and y8[step_count] > e and x8[step_count] > e
and c8_1[step_count] > e and c8_2[step_count] > e
and p8_1[step_count] > e and p8_2[step_count] > e
and p8_3[step_count] > e):
M = np.stack((z8, y8, x8, c8_1, c8_2, p8_1, p8_2, p8_3))
P = np.array([d_z, d_y, d_x, d_c1, d_c2])
if count < 4:
io.writeData(train_x, train_y, M, '8', threshold)
count = count + 1
else:
io.writeData(test_x, test_y, M, '8', threshold)
count = 0
io.writeParameters(parameters, P)
def gen(train_x, train_y, test_x, test_y, parameters, length, cutoff):
# Variable Declarations
step_count = length
threshold = cutoff
x7 = np.empty((step_count + 1, )) # tritrophic no omnivory
c7_1 = np.empty((step_count + 1, ))
c7_2 = np.empty((step_count + 1, ))
p7_1 = np.empty((step_count + 1, ))
p7_2 = np.empty((step_count + 1, ))
e = 0.000000001
dt = 0.01
min_x, max_x = (0.22, 0.30)
min_c1, max_c1 = (0.68, 0.78)
min_c2, max_c2 = (0.09, 0.19)
grid_fineness = 7
count = 5
# Initial Conditions for the food web topologies labeled 0 through 8. All start at 0.5
x7[0], c7_1[0], c7_2[0], p7_1[0], p7_2[0] = (0.5, 0.5, 0.5, 0.5, 0.5
) # tritrophic no omnivory
# Iterate through mortality rates
for d_x in np.linspace(min_x, max_x, grid_fineness):
for d_c1 in np.linspace(min_c1, max_c1, grid_fineness):
for d_c2 in np.linspace(min_c2, max_c2, grid_fineness):
# Evaulate time series
for i in range(step_count):
x7[i + 1], c7_1[i + 1], c7_2[i + 1], p7_1[i + 1], p7_2[
i + 1] = cm.tritrophicNoOmnivory(
x7[i], c7_1[i], c7_2[i], p7_1[i], p7_2[i], d_x,
d_c1, d_c2, dt)
# Check for coexistence and record the data if the species coexist
if (x7[step_count] > e and c7_1[step_count] > e
and c7_2[step_count] > e and p7_1[step_count] > e
and p7_2[step_count] > e):
M = np.stack((x7, c7_1, c7_2, p7_1, p7_2))
P = np.array([d_x, d_c1, d_c2])
if count < 4:
io.writeData(train_x, train_y, M, '7', threshold)
count = count + 1
else:
io.writeData(test_x, test_y, M, '7', threshold)
count = 0
io.writeParameters(parameters, P)
#d_c2
#d_c1
#d_x
# end gen()
def open_folder(self, folder=None):
if folder is None: folder = tk.filedialog.askdirectory()
Tests, title, A = FileIO.retrieve_data(self, folder)
if len(Tests) == 0:
self.error(8)
return
self.Tests = Tests
self.title = title
Functions.A = A
self.max_time = round(max([max(t.Time) for t in self.Tests]))
self.max_load = max([max(t.Load) for t in self.Tests])
self.min_load = min([min(t.Load) for t in self.Tests])
self.plot(self.title)
def buildDir(window):
# Prompt for a directory
tk.messagebox.showinfo(title="Open Data", message="Please select a directory for your data")
dir = tk.filedialog.askdirectory(parent=window.root)
# Initiate the parent tab
parent = Frame(window.nb)
window.nb.add(parent, text="Load/Save")
### Header and Buttons ###
Label(parent, text="Select Data", font='Helvetica 15 bold').pack(side='top')
Button(parent, command=lambda: window.widgets['tree'].delete(*window.widgets['tree'].get_children()),\
text="Clear Directories", takefocus=NO).pack(side='top')
select_call = lambda: window.error(7)\
if FileIO.populate_tree(window.widgets['tree'], tk.filedialog.askdirectory(parent=window.root)) else False
Button(parent, command=select_call, text="Select Directory", takefocus=NO).pack(side='top')
### Directory Tree ###
# Create tree
tFrame = Frame(parent)
tFrame.pack(side='top', expand=False)
window.widgets['tree'] = ttk.Treeview(tFrame, selectmode='browse')
window.widgets['tree'].column("#0", stretch=YES)
window.widgets['tree'].heading("#0", text="Directory")
window.widgets['tree'].pack(side='top', fill=BOTH, expand='yes')
# Populate tree
FileIO.populate_tree(window.widgets['tree'], dir)
# Add load data button
Button(parent, command=lambda: window.open_folder(window.widgets['tree'].focus()),\
text="Open Selected Data", takefocus=NO).pack(side='top')
### SAVE ###
ttk.Separator(parent, orient=HORIZONTAL).pack(pady=10, fill='x', anchor='ne')
Label(parent, text="Save Data", font="Helvetica 15 bold").pack(side=TOP)
# Save plot button
Button(parent, text="Save Plot", command=window.save_as, width=20).pack(side=TOP)
def touch_ktf_notify(t) :
try :
(id, level , s_type, email_list) = t
ok , path = home_fullpath(id)
if not os.path.exists(path) : DirUtil.make_dirs(path)
l = []
l.append("level=" + level + "&type=" + s_type)
for email in email_list :
l.append(email)
l.sort()
FileIO.write(os.path.join(path, '.ktf_notify'), l)
return (1, os.path.join(path, '.ktf_notify'))
except :
t, v, tb = sys.exc_info()
log = 'touch_ktf_notify(%s:%s)' % (t, v)
return (0, log)
def get_default_pins(settings, name):
exper_settings = FileIO.get_settings(
os.path.join(settings["projects"], name, name + '.json'))
analog = {
pin["pin_name"]: pin
for pin in exper_settings["pins"] if pin["type"] == pin_types["analog"]
}
digital = {
pin["pin_name"]: pin
for pin in exper_settings["pins"]
if pin["type"] == pin_types["digital"]
}
return analog, digital
def download(self, file, remote_name):
remote_path = os.path.join(self.path, remote_name)
remote_path = remote_path.replace("\\", "/")
handle = self.channel.file(remote_path, "rb")
downloaded = 0
for block in FileIO.read_blocks(handle, 16<<10):
downloaded += len(block)
logging.debug("Downloaded %d" % downloaded)
if self.progress_reporter is not None:
self.progress_reporter.set(downloaded)
file.write(block)
handle.close()
self.cleanup_connection()
def gen(train_x, train_y, test_x, test_y, parameters, length, cutoff):
# Variable Declarations
step_count = length
threshold = cutoff
x5 = np.empty((step_count + 1, )) # tritrophic omnivory C
c5_1 = np.empty((step_count + 1, ))
c5_2 = np.empty((step_count + 1, ))
p5_1 = np.empty((step_count + 1, ))
p5_2 = np.empty((step_count + 1, ))
e = 0.000000001
dt = 0.01
min_x, max_x = (0.26, 0.42)
min_c1, max_c1 = (0.51, 0.87)
min_c2, max_c2 = (0.01, 0.27)
grid_fineness = 7
count = 0
# Initial Conditions for the food web topologies labeled 0 through 8. All start at 0.5
x5[0], c5_1[0], c5_2[0], p5_1[0], p5_2[0] = (0.5, 0.5, 0.5, 0.5, 0.5
) # tritrophic omnivory C
# Iterate through mortality rates
for d_x in np.linspace(min_x, max_x, grid_fineness):
for d_c1 in np.linspace(min_c1, max_c1, grid_fineness):
for d_c2 in np.linspace(min_c2, max_c2, grid_fineness):
# Evaulate time series
for i in range(step_count):
x5[i + 1], c5_1[i + 1], c5_2[i + 1], p5_1[i + 1], p5_2[
i + 1] = cm.tritrophicOmnivoryC(
x5[i], c5_1[i], c5_2[i], p5_1[i], p5_2[i], d_x,
d_c1, d_c2, dt)
if (x5[step_count] > e and c5_1[step_count] > e
and c5_2[step_count] > e and p5_1[step_count] > e
and p5_2[step_count] > e):
M = np.stack((x5, c5_1, c5_2, p5_1, p5_2))
P = np.array([d_x, d_c1, d_c2])
if count < 4:
io.writeData(train_x, train_y, M, '5', threshold)
count = count + 1
else:
io.writeData(test_x, test_y, M, '5', threshold)
count = 0
io.writeParameters(parameters, P)
#d_c2
#d_c1
#d_x
# end gen()
def urban_positive(self):
urban_file = '../data/malawi_urban_extent.csv'
lines = FileIO.csv_reader(urban_file)
p_imgs_raw = []
for line in lines:
task_x = line['task_x']
task_y = line['task_y']
c = line['classification']
if c == 'urban_extent':
img = '%s-%s.jpeg' % (task_x, task_y)
p_imgs_raw.append(img)
p_imgs = list(set(p_imgs_raw))
return p_imgs
def gen(train_x, train_y, test_x, test_y, parameters, length, cutoff):
# Variable Declarations
step_count = length
threshold = cutoff
z0 = np.empty((step_count + 1, )) # five species chain
y0 = np.empty((step_count + 1, ))
x0 = np.empty((step_count + 1, ))
c0 = np.empty((step_count + 1, ))
p0 = np.empty((step_count + 1, ))
e = 0.000000001
dt = 0.01
d_z = 0.01 #Found through trial and error that all parameter combinations leading to coexistance had d_z = 0.01
min_y, max_y = (0.01, 0.09)
min_x, max_x = (0.01, 0.11)
min_c, max_c = (0.01, 0.11)
grid_fineness = 7
count = 0
# Initial Conditions for the food web topologies labeled 0 through 8. All start at 0.5
z0[0], y0[0], x0[0], c0[0], p0[0] = (0.5, 0.5, 0.5, 0.5, 0.5
) # five species chain
# Iterate through mortality rates
for d_y in np.linspace(min_y, max_y, grid_fineness):
for d_x in np.linspace(min_x, max_x, grid_fineness):
for d_c in np.linspace(min_c, max_c, grid_fineness):
# Evaulate time series
for i in range(step_count):
z0[i +
1], y0[i +
1], x0[i +
1], c0[i +
1], p0[i +
1] = cm.fiveSpeciesChain(
z0[i], y0[i], x0[i],
c0[i], p0[i], d_z, d_y,
d_x, d_c, dt)
# Check for coexistence and record the data if the species coexist
if (z0[step_count] > e and y0[step_count] > e
and x0[step_count] > e and c0[step_count] > e
and p0[step_count] > e):
M = np.stack((z0, y0, x0, c0, p0))
P = np.array([d_z, d_y, d_x, d_c])
if count < 4:
io.writeData(train_x, train_y, M, '0', threshold)
count = count + 1
else:
io.writeData(test_x, test_y, M, '0', threshold)
count = 0
io.writeParameters(parameters, P)
def __init__(self):
#attribute for total turns from FileIO
self.totalTurns = 0
#attributes for attack and defense types
self.probXLowAttack = 0
self.probLowAttack = 0
self.probMediumAttack = 0
self.probHighAttack = 0
self.probXLowDefense = 0
self.probLowDefense = 0
self.probMediumDefense = 0
self.probHighDefense = 0
# attributes for attack and defense tally types
self.tallyHighAttack = 0
self.tallyMediumAttack = 0
self.tallyLowAttack = 0
self.tallyXLowAttack = 0
self.tallyHighDefense = 0
self.tallyMediumDefense = 0
self.tallyLowDefense = 0
self.tallyXLowDefense = 0
# attributes for summary of results and turnbyturn
self.attackType = ""
self.defenseType = ""
self.result = ""
self.num = 1
self.totalBlocks = 0
self.totalHits = 0
# Manager creates 3 objects and calls the following:
self.aFileIO = FileIO()
self.aFileIO.readingFile()
self.assigningProbabiltiesTurns()
self.anAttacker = Attacker(self.probHighAttack,self.probMediumAttack, self.probLowAttack, self.probXLowAttack)
self.aDefender = Defender(self.probHighDefense, self.probMediumDefense, self.probLowDefense, self.probXLowDefense)
def load_and_use_model(model, all_infos, mapp):
(conceptsList, conceptsList_all, videofiles, needed_shots,
shot_paths) = all_infos
# init caffe net
net, transformer = init_caffe_net(model)
# read images to classify from folder
print "Preparing images for model " + model
all_images = read_images_for_predicting(transformer, shot_paths)
# perform classification
print "Classifying for model " + model
s_time = timeit.default_timer()
predicted_labels = classify(net, all_images)
time = format(timeit.default_timer() - s_time, '.4f')
FileIO.write_times('training_times/classification_times.csv', time, model,
'', len(predicted_labels))
print "Calculating evaluation parameters for model " + model
acc_values_for_all_concepts = calc_accuracy(predicted_labels, conceptsList,
mapp)
return acc_values_for_all_concepts
def MS_positive(self):
imgs = []
ms_file = '../data/project_922.csv'
lines = FileIO.read_lines(ms_file, 1)
for line in lines:
tmp = line.strip().split(',')
x, y = int(tmp[4]), int(tmp[5])
img = '%d-%d.jpeg' % (x, y)
yes_count, maybe_count, bad_img_count = int(tmp[8]), int(
tmp[9]), int(tmp[10])
if bad_img_count == 0 and (yes_count >= 2 or
(maybe_count + yes_count) >= 3):
imgs.append(img)
return imgs
def getStr2ObjectDict(filename, items):
"""
str(filename) -> list of CHSingleSent
"""
l = FileIO.readDataFromFile(filename)
i2css = {}
for item in l:
if not item[0] in items:
continue
try:
i2css[item[0]] = CHSingleSent(item)
except:
print "String Error:",item[0]
return i2css
def html_stuff(stitched_shots, all_shots, outstitched_shots, ground_truth_file, filename1, filename2, output):
data_prefix = 0
data_vars1 = 'var jumpoffs = ['
data_vars2 = 'var all_scenes = ['
data_vars3 = 'var jo = ['
data_suffix = 0
#read ground truth for shot boundaries
ground_truth_data = FileIO.read_shots_from_csv(ground_truth_file)
#get start- and end-frames
last_element = stitched_shots[len(stitched_shots)-1]
for i in stitched_shots:
a,b = ground_truth_data[i[0]]
if i is not last_element:
data_vars1 += str(a) + ', '
else:
data_vars1 += str(a) + '];\n'
last_element = all_shots[len(all_shots)-1]
for shot in all_shots:
a,b = shot
if shot is not last_element:
data_vars2 += str(a) + ', '
else:
data_vars2 += str(a) + '];\n'
#get start- and end-frames
last_element = outstitched_shots[len(outstitched_shots)-1]
for i in outstitched_shots:
a,b = ground_truth_data[i]
if i is not last_element:
data_vars3 += str(a) + ', '
else:
data_vars3 += str(a) + '];\n'
#read prefix and suffix (html and javascript code)
with open(filename1, 'r') as f1:
data_prefix = f1.read()
f1.close()
with open(filename2, 'r') as f2:
data_suffix = f2.read()
f2.close()
string = data_prefix + data_vars1 + data_vars2 + data_vars3 + data_suffix
with open(output, 'w') as f:
f.write(string)
f.close()
def upload(self, file, remote_name):
#print "Dummy uploading %s" % remote_name
#return
remote_path = os.path.join(self.path, remote_name)
remote_path = remote_path.replace("\\", "/")
handle = self.channel.file(remote_path, "wb")
uploaded = 0
for block in FileIO.read_blocks(file, 128<<10):
uploaded += len(block)
logging.debug("Uploaded %d" % uploaded)
if self.progress_reporter is not None:
self.progress_reporter.set(uploaded)
handle.write(block)
handle.close()
self.cleanup_connection()
def Part1():
# values = [100, 200, 500, 1000]
values = [100]
for v in values:
print "Points: " + str(v)
#optical flow and motion direction histogram calculation
# mdh_all = OM.createMotionDirectionHistograms('GroundTruth.csv', 'videos/oberstdorf08small.mp4', v, False, False)
# FileIO.save_histograms_to_file('mdh_8_' + str(v) + '.csv', mdh_all)
# print "Histograms created."
# #svm training and predicting
mdh_compl = FileIO.read_histograms_from_file('mdh_8_' + str(v) + '.csv')
accuracy, ITERATIONS, NF = SVM.svm_accuracy(mdh_compl)
print "average accuracy: " + str(accuracy/ITERATIONS/NF)
def read(filename) :
hash = {}
try :
(ok, content) = FileIO.full_read(filename)
if not ok : return ( 0, filename + ' not founc')
content = content.replace('\n\t', '')
lines = content.split('\n')
for line in lines :
v = line.rstrip().split(':')
if(len(v) != 2) : continue
hash[v[0].lower()] = v[1][1:]
return (1 , hash)
except :
t, v, tb = sys.exc_info()
print 'MailHeader read(%s)' % v
flag = 0
return (0, hash)
def writeDAT(self,delimiter=","):
"""Writes the STARS dat file.
writes projectName.dat file for STARS
"""
info = []
varNames = self.getAllVariableNames()
varNames.remove("csnames")
varNames.remove("csids")
header = [ len(varNames) ] + varNames
info.append(header)
for var in varNames:
v = self.varDict[var]
info.append( [v.name, v.type] )
vals = v.values
if v.type == "CSTS":
for id in range(self.n):
csYearVals = [ vals[year][id] for year in range(self.t) ]
info.append(csYearVals)
else:
info.append(vals[0])
datFile = self.createOutputFile(self.projPrefix, ".dat")
fdat = FIO.csvWriter(datFile, info, delimiter=" ")
def buildCorpus(dirPath, corpusName):
print('creating corpus!!');
tagClass = fetchLabel(dirPath);
classCnt = FileIO.countFiles(dirPath);
dictionary = {};
FileIO.wrtieToFile("corpus\classCount.txt", 'a', (tagClass + '\t' + str(classCnt) + '\n' ));
for dir_entry in os.listdir(dirPath):
text = FileIO.readFile(os.path.join(dirPath, dir_entry));
text = Tokenizer.tokenizer(text);
for token in text.split('\n'):
if token not in dictionary:
dictionary[token] = {};
if tagClass not in dictionary[token]:
dictionary[token][tagClass] = 0;
dictionary[token][tagClass] = dictionary[token][tagClass] + 1;
for key, value in dictionary.items():
FileIO.wrtieToFile(corpusName, 'a', (key + '\t' + str(value[tagClass]) + '\t' + tagClass + '\n'));
print('Corpus creation : Done..');
def buildBinarizedCorpus(dirPath, corpusName):
print('creating binarized corpus!!');
tagClass = fetchLabel(dirPath);
classCnt = FileIO.countFiles(dirPath);
FileIO.wrtieToFile("corpus\classCount.txt", 'a', (tagClass + '\t' + str(classCnt) + '\n' ));
corpusDict = {};
for dir_entry in os.listdir(dirPath):
fileTokens = {};
text = FileIO.readFile(os.path.join(dirPath, dir_entry));
text = Tokenizer.tokenizer(text);
for token in text.split('\n'):
if token not in fileTokens:
fileTokens[token] = 1;
if token not in corpusDict:
corpusDict[token] = {};
corpusDict[token][tagClass] = 1;
else:
corpusDict[token][tagClass] = corpusDict[token][tagClass] + 1;
for key, value in corpusDict.items():
FileIO.wrtieToFile(corpusName, 'a', (key + '\t' + str(value[tagClass]) + '\t' + tagClass + '\n' ));
print('binarized corpus creation done!!');
