def getValidationData(self): if self.x_valid is None: x_tmp = [] y_tmp = [] print("Starting to load validation sets ") for c, files in self.trainingsets.iteritems(): y = self.testsets.keys().index( c) #We use the testset as a reference for file in files: pics = cv2.imread(self.path + c + '/' + file, cv2.CV_LOAD_IMAGE_GRAYSCALE ) #unit8 from e.g. 34 to 255 x_tmp.append(np.reshape( pics / 255., len(pics)**2)) #To floats from 0 to 1 y_tmp.append(y) print("Finished, loading validation sets") perm = np.random.permutation(len(y_tmp)) self.x_valid = theano.shared(np.asarray( x_tmp, theano.config.floatX)[perm], borrow=True) self.y_valid = T.cast( theano.shared(np.asarray(y_tmp, theano.config.floatX)[perm], borrow=True), 'int32') return self.x_valid, self.y_valid
def post(self, *args, **kwargs): self.adminAuthenticated() db = self.getDBSession() appId = self.get_argument('appId', '') if len(appId): self.app = db.query(App).filter(App.appId == appId).first() if not self.app: raise RuntimeError('Can`t find app for editing') else: self.app = App() name = self.get_argument('name', '') code = self.get_argument('code', '') errors = [] if len(name) == 0: errors.append(u'Название не должно быть пустое') else: self.app.name = name if len(code) == 0: errors.append(u'Код не должен быть пустым') else: self.app.code = code if not len(errors): db.merge(self.app) db.commit() self.redirect('/admin/apps/') else: self.run()
def get_top_m_n_grams(m, n_gram): """Reduces a list of n_grams with word sequences and their frequencies to the top *m* ones. :param m: amount of sequences and corresponding frequencies to return :param n_gram: an object with attribues 'sequencies' and (index-wise) corresponding 'frequencies' :returns: The top *m* sequences (and their frequencies) of the input n-gram in a descending order. """ top_m_freqs = [0] top_m_index = [0] for seq in n_gram["sequences"]: freq = n_gram["frequencies"][n_gram["sequences"].index(seq)] for i in range(0, len(top_m_freqs)): if freq > top_m_freqs[i]: top_m_freqs.insert(i, freq) top_m_index.insert(i, n_gram["sequences"].index(seq)) if len(top_m_freqs) > m: top_m_freqs.pop() top_m_index.pop() break result = { "sequences": [], "frequencies": [] } for index in top_m_index: result["sequences"].append(n_gram["sequences"][index]) result["frequencies"].append(n_gram["frequencies"][index]) return result
def giveMeNewTraining(self): x_tmp = [] y_tmp = [] print("Starting to create new training data ") for c, files in self.trainingsets.iteritems(): y = self.testsets.keys().index( c) #We use the testset as a reference for file in files: image = cv2.imread( self.path + c + '/' + file, cv2.CV_LOAD_IMAGE_GRAYSCALE) #unit8 from e.g. 34 to 255 image2 = expandTraining.distorb(image / 255.) # cv2.imshow('org', cv2.resize(image, (280, 280))) # cv2.imshow('mani', cv2.resize(image2, (280, 280))) # cv2.waitKey(2000) x_tmp.append(np.reshape( image2, len(image2)**2)) #To floats from 0 to 1 y_tmp.append(y) print("Finished, creating new training data") perm = np.random.permutation(len(y_tmp)) return theano.shared(np.asarray(x_tmp, theano.config.floatX)[perm], borrow=True), T.cast( theano.shared(np.asarray( y_tmp, theano.config.floatX)[perm], borrow=True), 'int32')
def rule_one_neg(text, sentence, pos_tagged): _dict = {} text = [] for word_tagged in pos_tagged: _dict[word_tagged[0]] = word_tagged[1] if len(word_tagged[0]) > 1: if (word_tagged[0] not in stopwords_set) or (word_tagged[0] in negation): text.append(word_tagged[0]) ret_list = [] i = 0 while i < len(text): word_set = text[i:i + 3] if len(set(word_set).intersection(negation)) > 0 and len(word_set) is 3: lis = [] for word in word_set: lis.append((word, _dict[word])) if lis[1][1] in adj and lis[2][1] in noun: stri = lis[0][0] + " " + lis[1][0] + " " + lis[2][0] ret_list.append(stri) if lis[0][1] in adj and lis[1][1] in noun: stri = lis[2][0] + " " + lis[0][0] + " " + lis[1][0] ret_list.append(stri) i += 1 return ret_list
def compute_most_probable_digraph(so_char_analysis, start): """Recursively computes the most probable digraph given a second-order correlation matrix as input and a character to start with. When the successor of the start character is found all possible successors of this character are removed in order to avoid loops.""" characters = so_char_analysis["characters"] frequencies = so_char_analysis["frequencies"] if len(characters) == 0: return start else: current_char = start next_char = "" next_max_freq = 0 remove_list = [] for i in range (0, len(characters)): if characters[i].startswith(current_char): remove_list.append(characters[i]) if frequencies[i] > next_max_freq: next_max_freq = frequencies[i] next_char = characters[i][1] for i in range(0, len(remove_list)): index = characters.index(remove_list[i]) characters.remove(characters[index]) frequencies.remove(frequencies[index]) return start + compute_most_probable_digraph({ "characters": characters, "frequencies": frequencies }, next_char)
def main(): # prepare data trainingData = [] testData = [] split = 0.67 loadDataset('data/train.csv', split, trainingData, testData) # loadDataset('actualdata/train.csv', split, trainingData, testData) matches = 0 for x in range(len(testData)): instance = testData[x] neighbors = getNeighbors(trainingData,instance,3) guess = int(getGuessedDigit(neighbors)) actual = int(instance[0]) echo('(%d) Guessed Value %d ' %(x , guess)) echo('(%d) Actual Value %d ' %(x , actual)) if(guess == actual): matches +=1 else: echo('Displaying character which is not matching') # plt.drawPixels(instance[1:], nRows=28, nCols=28) echo ('Total matches %d out of %d' %(matches,len(testData))) echo ('Accuracy is %f ' %((matches)/float(len(testData))*100)+'%')
def ber_calc(init_frame, end_frame): error = 0 for x in range(0, len(init_frame)): if init_frame[x] != end_frame[x]: error += 1 ber = (error / len(init_frame)) return ber
def read_Soil_Data_From_Wise3_Horizon_File(in_csv): WISE_HORIZON_RECORD = [] WISE_HORIZON_DATA_LIST_RECORD = [] WISE_HORIZON_DATA_LIST_LAYER_BY_RECORD = [] try: f = open(in_csv, "rb") l = UnicodeReader(f) l.next() consider_id = "" for row in l: WISE3_ID = str(row[0]).upper() WISE3_honu = str(row[1]).upper().strip() if (str(row[3]).strip() is not None and str(row[3]).strip() != ""): WISE3_topdep = float(row[3]) else: WISE3_topdep = float(0) if (str(row[4]).strip() is not None and str(row[4]).strip() != ""): WISE3_botdep = float(row[4]) else: WISE3_botdep = float(0) if (str(row[7]).strip() is not None and str(row[7]).strip() != ""): WISE3_orgc = float(row[7]) else: WISE3_orgc = float(-9.9) if (str(row[23]).strip() is not None and str(row[23]).strip() != ""): WISE3_sand = float(row[23]) else: WISE3_sand = float(-9.9) if (str(row[24]).strip() is not None and str(row[24]).strip() != ""): WISE3_silt = float(row[24]) else: WISE3_silt = float(-9.9) if (str(row[25]).strip() is not None and str(row[25]).strip() != ""): WISE3_clay = float(row[25]) else: WISE3_clay = float(-9.9) if (str(row[27]).strip() is not None and str(row[27]).strip() != ""): WISE3_bulkdens = float(row[27]) else: WISE3_bulkdens = float(-9.9) if (WISE3_ID != consider_id): if (WISE_HORIZON_DATA_LIST_LAYER_BY_RECORD is not None and len(WISE_HORIZON_DATA_LIST_LAYER_BY_RECORD) > 0): WISE_HORIZON_DATA_LIST_RECORD.append(WISE_HORIZON_DATA_LIST_LAYER_BY_RECORD) WISE_HORIZON_DATA_LIST_LAYER_BY_RECORD = [] consider_id = WISE3_ID WISE_HORIZON_RECORD = [WISE3_ID,WISE3_honu,WISE3_topdep,WISE3_botdep,WISE3_orgc,WISE3_sand,WISE3_silt,WISE3_clay,WISE3_bulkdens] WISE_HORIZON_DATA_LIST_LAYER_BY_RECORD.append(WISE_HORIZON_RECORD) else: WISE_HORIZON_RECORD = [WISE3_ID,WISE3_honu,WISE3_topdep,WISE3_botdep,WISE3_orgc,WISE3_sand,WISE3_silt,WISE3_clay,WISE3_bulkdens] WISE_HORIZON_DATA_LIST_LAYER_BY_RECORD.append(WISE_HORIZON_RECORD) if (WISE_HORIZON_DATA_LIST_LAYER_BY_RECORD is not None and len(WISE_HORIZON_DATA_LIST_LAYER_BY_RECORD) > 0): WISE_HORIZON_DATA_LIST_RECORD.append(WISE_HORIZON_DATA_LIST_LAYER_BY_RECORD) return WISE_HORIZON_DATA_LIST_RECORD except Exception, err: print err return -1
def merge_sort(A): if len(A) <= 1: return A middle = int(len(A) / 2) left = merge_sort(A[:middle]) right = merge_sort(A[middle:]) return merge(left, right)
def calculate_Normalization(BIG_LIST,entry_plot_data_numberous): print "=== Normalization Calculation=====" datasets = [] for i in range(0,len(BIG_LIST)): datasets.append(BIG_LIST[i][3]) #print "Before : " #print datasets[0] max_list_value = [] min_list_value = [] for i in range(0,len(datasets[0])): max_list_value.append(-9999999999999999999999999999) min_list_value.append(9999999999999999999999999999) for record in datasets: for i in range(0,len(record)): if (float(record[i]) > max_list_value[i]): max_list_value[i] = float(record[i]) if (float(record[i]) < min_list_value[i]): min_list_value[i] = float(record[i]) for i in range(0,len(entry_plot_data_numberous)): if (float(entry_plot_data_numberous[i]) > max_list_value[i]): max_list_value[i] = float(entry_plot_data_numberous[i]) if (float(entry_plot_data_numberous[i]) < min_list_value[i]): min_list_value[i] = float(entry_plot_data_numberous[i]) entry_plot_data_numberous[i] = (float(entry_plot_data_numberous[i]) - float(min_list_value[i])) / (float(max_list_value[i]) - float(min_list_value[i])) print "Max List : " print max_list_value print "Min List : " print min_list_value for record in datasets: for i in range(0,len(record)): new_record_value = (float(record[i]) - float(min_list_value[i])) / (float(max_list_value[i]) - float(min_list_value[i])) record[i] = new_record_value
def test_process_negative_edge(self): # outer = 148, left half (first half of data though!) xds, yds = self.getInnerAndDetDatasetForGivenOuterValue(148.0) x = dnp.asarray(xds.data[:(len(xds) / 2)]) y = dnp.asarray(yds.data[:(len(yds) / 2)]) expected = 0.000000, 0.000000, -3.896408, 0.006343, 1.699449, 0.006343 check(TwoGaussianEdges()._process(x, y), expected, TwoGaussianEdges().labelList)
def test_process_positive_edge(self): # outer = 148, right half (second half of data though!) xds, yds = self.getInnerAndDetDatasetForGivenOuterValue(148.0) x = dnp.asarray(xds.data[(len(xds) / 2):]) y = dnp.asarray(yds.data[(len(yds) / 2):]) expected = -3.944682, 0.006067, 0.000000, 0.000000, 1.683015, 0.006067 check(TwoGaussianEdges()._process(x, y), expected, TwoGaussianEdges().labelList)
def function_to_run(review): ret_list = [] for _, row in review.iterrows(): _scores_ = {} del row['_id'] tags = row['final'] if len(set(tags.keys())) > 0: for key in tags.keys(): _scores_[key] = {} word = key.split(" ") for category in categories.keys(): cat = categories[category] try: _scores_[key][category] = sum( word_vectors.n_similarity(word, cat)) except Exception as e: break pass row['score'] = _scores_ ret_list.append(row) if len(ret_list) > 10000: df = pd.DataFrame(ret_list) to_mongo_db(df, 'yelp_review_scored_pair_all_not_final') print("Written to DB", len(ret_list), 'time from start', (time.time() - start_time)) ret_list = [] df = pd.DataFrame(ret_list) to_mongo_db(df, 'yelp_review_scored_pair_all_not_final') print("Written to DB", len(ret_list), 'time from start', (time.time() - start_time))
def merge_sort(A): if len(A)<=1: return A middle = int(len(A)/2) left = merge_sort(A[:middle]) right = merge_sort(A[middle:]) return merge (left,right)
def parse_line(self, line): tag = {} m = re.search('(?P<indent>[\s\t]*)//@([\s\t]*)(?P<keyword>[a-zA-Z0-9_]+)(?P<info>.*)', line) if (m != None): keyword = m.group('keyword') tag["keyword"] = keyword data = {} indent = m.group('indent') if (indent != None): tag['indent'] = len(indent) else: tag['indent'] = 0 info = m.group('info') while (info and len(info) > 0): m = re.search('(?P<key>[a-zA-Z0-9_]+)(=?)(?P<value>[^\s]*)(?P<info>.*)', info) if (m != None): key = m.group('key') value = m.group('value') data[key] = value info = m.group('info') else: info = None tag["data"] = data #print tag return tag
def create_reference(sis_to_mean): meanpath = settings.referencefile for f in sis_to_mean[:]: if not f.endswith(".sis"): sis_to_mean.remove(f) if len(sis_to_mean) > 0: tot_K = None tot_Na = None for file in sis_to_mean: K, Na = loadimg(file) if tot_K != None: tot_K += K else: tot_K = K if tot_Na != None: tot_Na += Na else: tot_Na = Na tot_K /= len(sis_to_mean) tot_Na /= len(sis_to_mean) tot = numpy.concatenate( (numpy.ma.filled(tot_K, 0), numpy.ma.filled(tot_Na, 0))) tot = (tot + 1) * 6553.6 write_raw_image(meanpath, tot)
def parse_line(self, line): tag = {} m = re.search( '(?P<indent>[\s\t]*)//@([\s\t]*)(?P<keyword>[a-zA-Z0-9_]+)(?P<info>.*)', line) if (m != None): keyword = m.group('keyword') tag["keyword"] = keyword data = {} indent = m.group('indent') if (indent != None): tag['indent'] = len(indent) else: tag['indent'] = 0 info = m.group('info') while (info and len(info) > 0): m = re.search( '(?P<key>[a-zA-Z0-9_]+)(=?)(?P<value>[^\s]*)(?P<info>.*)', info) if (m != None): key = m.group('key') value = m.group('value') data[key] = value info = m.group('info') else: info = None tag["data"] = data #print tag return tag
def test(): println('GO pyparse.parser.endless_sky') name = 'EndlessSkyParser with valid data' try: s = """# comment root1 token1 "token 2" `token 3` child1.1 token 1.1 child1.2 root2 # ignored """ p = EndlessSkyParser(s) node = p.maybe(p.rule_DataNode) assert node is not None, node assert node.indent == '', node assert node.tokens == ( 'root1', 'token1', '"token 2"', '`token 3`', ), node assert node.lines == slice(1, 5), node # includes the comment assert len(node.children) == 2, node assert node.children[0].indent == ' ', node assert node.children[0].tokens == ( 'child1.1', 'token', '1.1', ), node assert node.children[0].lines == slice(3, 4), node assert len(node.children[0].children) == 0, node assert node.children[1].indent == ' ', node assert node.children[1].tokens == ('child1.2', ), node assert node.children[1].lines == slice(4, 5), node assert len(node.children[1].children) == 0, node node = p.maybe(p.rule_DataNode) assert node is not None, node assert node.indent == '', node assert node.tokens == ('root2', ), node assert node.lines == slice(5, 7), node # includes the empty line assert len(node.children) == 0, node node = p.maybe(p.rule_DataNode) assert node is None, node # ignores the rest println('PASS', name) except: println('FAIL', name) raise # print stacktrace # TODO test invalid data # TODO test s? println('OG pyparse.parser.endless_sky')
def handle_data(self, data): try: if "LoggerAccesibility" in data: inicio = data.find("LoggerAccesibility") sub = data[inicio + len('LoggerAccesibility') + 1 + len('http://') + 1:] fin = sub.find(',true') - 1 self.server = sub[:fin] except: ui.message("Error en handle data")
def readpathfile(): with open(pathfilename) as f: possiblepaths = f.readlines() length = len(possiblepaths) """splitting strings in sublists and converting to int""" for i in range(length): possiblepaths[i] = possiblepaths[i].split() for i in range(length): for j in range(len(possiblepaths[i])): possiblepaths[i][j] = int(possiblepaths[i][j]) return possiblepaths, length
def check_compatible(time,lat,long): if (len(time) == 366 or len(time) == 365): if (len(long) == 1440 and len(lat) == 720): if (lat[0] == 89.875 and lat[719] == -89.875): if (long[0] == 0.125 and long[1439] == 359.875): return 1 else: return -1 else: return -1 else: return -1
def readfiles(): """this is the empty content list""" """numbers are surrounded by a 'wall' = 99""" content = [[99, 99, 99, 99, 99, 99], [99, 0, 0, 0, 0, 99], [99, 0, 0, 0, 0, 99], [99, 0, 0, 0, 0, 99], [99, 0, 0, 0, 0, 99], [99, 99, 99, 99, 99, 99]] """open file and reading file from tesser, skipping empty lines""" with open('%s.txt' % (outfilename)) as f_in: contenttemp = (line.rstrip() for line in f_in) contenttemp = list(line for line in contenttemp if line) # Non-blank contenttemp in a list f_in.close() length = len(contenttemp) print len(contenttemp[0]) """splitting every element in more elements, creating a 2D Matrix""" for i in range(0, length): contenttemp[i] = contenttemp[i].split() """raising TypeError if length of contenttemp is larger then 4, if OCR is successful, no Error here""" for i in range(0, length): if (len(contenttemp[i]) > 4): raise TypeError """integrating contenttemp to the already created content list""" for index in range(4): for innerindex in range(4): content[index + 1][innerindex + 1] = contenttemp[index][innerindex] """converting content to int""" for index in range(1, 5): for innerindex in range(1, 5): content[index][innerindex] = int(content[index][innerindex]) """raise TypeError if a 0 exists in content, if OCR is successful, no Error here""" for x in range(1, 5): for y in range(1, 5): if (content[x][y] == 0): raise TypeError """raises TypeError if 'border' is not 99, if OCR is successful, no Error here""" for y in range(0, 5): if (content[0][y] != 99): print "true" raise TypeError for y in range(0, 6): if (content[5][y] != 99): print "true" raise TypeError for x in range(0, 6): if (content[x][5] != 99): print "true" raise TypeError for x in range(0, 6): if (content[x][5] != 99): print "true" raise TypeError return content
def initialize_third_order_matrix(): """Initializes a third-order matrix [2x64000].""" characters = [] frequencies = [] for i in range(0, len(alphabet)): for j in range(0, len(alphabet)): for k in range(0, len(alphabet)): characters.append(alphabet[i] + alphabet[j] + alphabet[k]) frequencies.append(0) return { "characters": characters, "frequencies": frequencies }
def __exit__(self, exc_type, exc_value, traceback): r""" Keep or revert parser state. """ if not isinstance(self, ParserSkeleton): raise RuntimeError, type(self) # expecting ParserSkeleton if not len(self.stack) > 0: raise RuntimeError, len(self.stack) # expecting a non-empty stack _state = self.stack.pop(-1) if exc_type is None and exc_value is None and traceback is None: # all ok, forget previous state pass else: # has exception, revert to previous position and propagate exception self.state = _state
def main(): if RECORDER_NAME.upper() == "ALL": print "Developed LandInfo Photo" if (LAST_DATE is None) or (len(LAST_DATE) <> 8) or not int(LAST_DATE): last_update_date = support_rhm_sync_data.get_last_update_date() if (last_update_date is None) or (len(last_update_date) <> 8): last_update_date = "20150101" else: last_update_date = LAST_DATE print "\n========================================\n" print "\n===Start Working====\n" print last_update_date print "\n========================================\n" # last_update_date = "20140102" # Request to Nasim get data lst_records = requestPlotPhotos.getListPhotoUrls(last_update_date) # print last_update_date # For loop all result of Nasim to insert our database for record in lst_records: try: print ("===Consider : %s \n" % (last_update_date)) print record print ("\n") landinfor_name = str(record[0]) landinfor_recorder_name = str(record[1]) landinfor_landscapeNorthPhotoURL = str(record[4]) landinfor_landscapeEastPhotoURL = str(record[5]) landinfor_landscapeSouthPhotoURL = str(record[6]) landinfor_landscapeWestPhotoURL = str(record[7]) landinfor_soilPitPhotoURL = str(record[2]) landinfor_soilSamplesPhotoURL = str(record[3]) result = support_rhm_sync_data.update_photos_landinfo( landinfor_name, landinfor_recorder_name, landinfor_landscapeNorthPhotoURL, landinfor_landscapeEastPhotoURL, landinfor_landscapeSouthPhotoURL, landinfor_landscapeWestPhotoURL, landinfor_soilPitPhotoURL, landinfor_soilSamplesPhotoURL, ) if result == 1: print "\n ===Update successfully===" else: print "\n ===Error ----" except: pass else: print "Does not support now !"
def merge(left,right): result=[] while len(left)>0 and len(right)>0: if left[0][0]<= right[0][0]: result.append(left[0]) left=left[1:] else: result.append(right[0]) right=right[1:] if len(left)>0: result+=left if len(right)>0: result+=right return result
def break_on_5(list): list = send_index(list) start = 0 res = [] while start < len(list): if start + 5 <= len(list) - 1: end = start + 5 res.append(merge_sort_pivot(list[start:end])) start = end else: end = len(list) res.append(merge_sort_pivot(list[start:end])) start = end return merge_sort_pivot(res)
def getNeighbors(trainingSet, testInstance, k): distances = [] length = len(testInstance) - 1 for x in range(len(trainingSet)): if(x==0): continue dist = euclideanDistance(testInstance, trainingSet[x], length) # distances.append((trainingSet[x], dist,trainingSet[x][0])) distances.append((dist,trainingSet[x])) distances.sort(key=operator.itemgetter(0)) neighbors = [] for x in range(k): neighbors.append(distances[x]) return neighbors
def break_on_5(list): list=send_index(list) start =0 res = [] while start<len(list): if start+5<=len(list)-1: end=start+5 res.append(merge_sort_pivot(list[start:end])) start=end else: end = len(list) res.append(merge_sort_pivot(list[start:end])) start=end return merge_sort_pivot(res)
def find_data_2(data,key): try: intLength = len(data) for i in range(0, intLength-3): line = data[i] if (line is None or len(line) < 40): continue line = line[0:40] if (checking_key(line, key) == 1): return [line,i] return None except Exception, err: print err pass
def merge(left, right): result = [] while len(left) > 0 and len(right) > 0: if left[0][0] <= right[0][0]: result.append(left[0]) left = left[1:] else: result.append(right[0]) right = right[1:] if len(left) > 0: result += left if len(right) > 0: result += right return result
def get_dirsize_Byte(self, dir): """Get a directory of the total number of bytes.""" if len(dir) <= 0: print "get a directory of the total number of bytes params error" return 0 for root, dirs, files in os.walk(dir): print "dir is ", dirs, files try: self.allsizebyte += sum(getsize(join(root,name)) for name in files) self.file_countnum += len(files) except: print sys.exc_info()[0],sys.exc_info()[1] pass print "dirsize is %d filenum is %d", self.allsizebyte, self.file_countnum return self.allsizebyte, self.file_countnum
def get_number_layers(ID): lst_rock_values = get_list_value_of_rock_fragment_to_consider_number_layer(ID) length_lst_rock_values = len(lst_rock_values) lst_texture_values = get_values_texture_for_soil_horizon_to_consider_number_layers(ID) length_lst_texture_values = len(lst_texture_values) if (( not int("0") in lst_rock_values) or (not int("0") in lst_texture_values)): return MAXIMUM_NUMBER_LAYERS elif ((lst_rock_values[length_lst_rock_values-1] == 1) or (lst_texture_values[length_lst_texture_values-1] == 1)): return MAXIMUM_NUMBER_LAYERS else: for i in reversed(range(0,MAXIMUM_NUMBER_LAYERS)): if ((lst_rock_values[i] == 1) and (lst_rock_values[i+1] == 0)): return (i+1) return MAXIMUM_NUMBER_LAYERS
def _codepoint(self, low, high=None): if high is None: high = low if not low <= high: raise ValueError, ( low, high, ) # expecting low <= high _p = Utf8Parser(self.state.data) _codepoint = _p.rule_Codepoint() _codepoint = _codepoint.codepoint _n = len(self.state.data) - len(_p.state.data) return ( _codepoint, self.consume(_n), )
def token_HEX(self): r""" hex '0' . '9' 'A' . 'F' """ _c = self[0] assert _c is not None if _c >= '0' and _c <= '9': return self.consume(len(_c)) if _c >= 'A' and _c <= 'F': return self.consume(len(_c)) assert False, ( _c, self.state, ) # not a token_HEX?
def arange(n, *xs, **kwds): if 'dtype' in kwds: elt_t = _get_type(kwds['dtype']) del kwds['dtype'] else: elt_t = Int64 assert len( kwds) == 0, "Unexpected keyword arguments to 'arange': %s" % kwds array_t = make_array_type(elt_t, 1) count = __builtin__.len(xs) assert 0 <= count <= 2, "Too many args for range: %s" % ((n, ) + tuple(xs)) if count == 0: start = zero_i64 stop = n step = one_i64 elif count == 1: start = n stop = xs[0] step = one_i64 else: start = n stop = xs[0] step = xs[1] if elt_t != Int64: start = Cast(start, type=elt_t) stop = Cast(stop, type=elt_t) step = Cast(step, type=elt_t) return Range(start, stop, step, type=array_t)
def filter_npmjs_node_modules_path_of(till, dirpath): re = '' while dirpath != till and __builtin__.len(dirpath) > 0: if os.path.basename(dirpath) != 'node_modules': re = os.path.join(os.path.basename(dirpath), re) dirpath = os.path.dirname(dirpath) return re
def __call__(self, *args,**kwargs): r = self.sendToHub(('EXP_DEVICE','DEV_RPC',self.device_class,self.method_name,args,kwargs)) r=r[1:] if len(r)==1: return r[0] else: return r
def author_attribution(text): """First, computes the second-order matrix (relative frequencies) of the input text. Second, computes the sum of the quadratic deviations of the input text against each author in the database.""" text_char_analysis = analyze_text(text, 2, False) freqs_text = abs_to_rel_freq(text_char_analysis, 2) authors = open("res/authors/index").readlines() best_match_sum = maxint best_match_author = "" for author in authors: author = author.replace("\n", "") current_sum = 0 author_char_analysis = js.load(open("res/authors/" + author)) freqs_author = author_char_analysis["frequencies"] for i in range(0, len(freqs_author)): current_sum += (freqs_text[i] - freqs_author[i]) ** 2 print "Text matches " + author + " by " + str(current_sum) if current_sum < best_match_sum: best_match_sum = current_sum best_match_author = author return best_match_author
def compute_n_gram_words(n, text, n_gram=None): """Returns *n*-grams of a given *text* source :param n: the size of the contiguous sequence of words :param text: the text source to be analyzed :param n_gram: takes an existing n_gram as input, which will be extended :returns: an object n_gram: { 'sequences': [*top **m** **n**-grams*], 'frequencies': [*the frequencies of n-grams*] } """ if n_gram == None: n_gram = { "sequences": [], "frequencies": [] } if n == 1: total_words = 0 for word in text.split(" "): total_words += 1 for special_char in special_characters: word = word.replace(special_char, "") word = word.lower() if word not in n_gram["sequences"]: n_gram["sequences"].append(word) n_gram["frequencies"].append(1) else: n_gram["frequencies"][n_gram["sequences"].index(word)] += 1 for i in range(0,len(n_gram["frequencies"])): n_gram["frequencies"][i] = n_gram["frequencies"][i] / (0.0 + total_words) else: #TODO pass return n_gram
def test_arp_table(): ssh('arp -n') read_cp_arp_table() debug_print('ARP table from Linux:') linux_arp_dict = {} fout = open(fname, 'r') for line in fout: if len(line.strip()) > 0: split_line = line.strip().split() if split_line[0] != 'Address' and split_line[1] != '(incomplete)': net_addr = split_line[0].strip() mac_addr = split_line[2].strip().upper() debug_print( "net address: %s , mac address: %s" %(net_addr, mac_addr)) linux_arp_dict[net_addr] = mac_addr if net_addr not in cp_arp_dict: print '*** ERROR: entry %s %s not in CP ARP table' %(net_addr , linux_arp_dict[net_addr]) exit(1) else: if mac_addr != cp_arp_dict[net_addr]: print '*** ERROR: entry in linux table not equal to CP ARP table. %s != %s' %(mac_addr , cp_arp_dict[net_addr]) exit(1) for key in cp_arp_dict: if key not in linux_arp_dict: print '*** ERROR: entry %s %s not in Linux ARP table' %(key,cp_arp_dict[key]) exit(1) else: if cp_arp_dict[key] != linux_arp_dict[key]: print '*** ERROR: entry not equal in Linux and CP ARP tables. %s != %s' %(cp_arp_dict[key] , linux_arp_dict[key]) exit(1) debug_print("ARP table test OK")
def get_first_picture(self): """ Returns a tuple containing the absolute path to the first image and the angle at which it was taken :return: A tuple containing of the absolute image path and the angle (path, angle) """ return self.pictures[0] if len(self.pictures) > 0 else None
def create_val_and_train(df, seed, ids, split_rt=0.20): """ Creates two samples (generally used to create train and validation samples) Parameters ---------------------------------------------------- ids: this is the level of randomization, so if you want to randomize countries, rather than cities, you would set this to 'countries' split_rate: pct of data to assign as validation Output ---------------------------------------------------- trn_for_mods (1-split_rate of df), trn_for_val (split_rate of data) """ np.random.seed(seed) # Get vector of de-dupped values of ids id_dat = pd.DataFrame(df[ids].drop_duplicates()) # Create random vector to split train val on vect_len = len(id_dat.ix[:, 0]) id_dat["rand_vals"] = np.array(np.random.rand(vect_len, 1)) df = pd.merge(df, id_dat, on=ids) # split data into two dfs trn_for_mods = df[df.rand_vals > split_rt] trn_for_val = df[df.rand_vals <= split_rt] # drop rand_vals trn_for_val = trn_for_val.drop("rand_vals", axis=1) trn_for_mods = trn_for_mods.drop("rand_vals", axis=1) return trn_for_mods, trn_for_val
def rule_DataLine(self): r""" rule_DataLine = token_WHITESPACE? ( token_DATA token_WHITESPACE? )+ rule_EmptyLine ;; the token_WHITESPACE at the start is the indent """ with self: # revert state on error _s = [] _lineNumber = self.state.lineNumber _indent = self.maybe(self.token_WHITESPACE) if _indent is None: _indent = self.consume(0) _s += [_indent] _tokens = [] while True: try: _token = self.token_TOKEN() _tokens += [_token] _s += [_token] _whitespace = self.maybe(self.token_WHITESPACE) if _whitespace is not None: _s += [_whitespace] except AssertionError: break # no more tokens _empty_line = self.rule_EmptyLine() _s += [_empty_line.s] assert len(_tokens) > 0 # expecting a token _s = tuple(_s) return self.DataLine(s=_s, indent=_indent, tokens=_tokens, lineNumber=_lineNumber)
def getValidationData(self): if self.x_valid is None: x_tmp = [] y_tmp = [] print("Starting to load validation sets ") for c,files in self.trainingsets.iteritems(): y = self.testsets.keys().index(c) #We use the testset as a reference for file in files: pics = cv2.imread(self.path + c + '/' + file, cv2.CV_LOAD_IMAGE_GRAYSCALE) #unit8 from e.g. 34 to 255 x_tmp.append(np.reshape(pics / 255., len(pics)**2)) #To floats from 0 to 1 y_tmp.append(y) print("Finished, loading validation sets") perm = np.random.permutation(len(y_tmp)) self.x_valid = theano.shared(np.asarray(x_tmp, theano.config.floatX)[perm],borrow=True) self.y_valid = T.cast(theano.shared(np.asarray(y_tmp, theano.config.floatX)[perm],borrow=True), 'int32') return self.x_valid, self.y_valid
def len_(iterable): import __builtin__ try: return __builtin__.len(iterable) except TypeError: count = 0 for i in iterable: count+=1 return count
def arange(n, *xs, **kwds): if 'dtype' in kwds: elt_t = _get_type(kwds['dtype']) del kwds['dtype'] else: elt_t = Int64 assert len(kwds) == 0, "Unexpected keyword arguments to 'arange': %s" % kwds array_t = make_array_type(elt_t, 1) count = __builtin__.len(xs) assert 0 <= count <= 2, "Too many args for range: %s" % ((n,) + tuple(xs)) if count == 0: start = zero_i64 stop = n step = one_i64 elif count == 1: start = n stop = xs[0] step = one_i64 else: start = n stop = xs[0] step = xs[1] if elt_t != Int64: start = Cast(start, type = elt_t) stop = Cast(stop, type = elt_t) step = Cast(step, type = elt_t) return Range(start, stop, step, type = array_t)
def get_collection_value_from_all_layer_texture(LIST_TEXTURE): try: SAND_LIST = [0,0,0,0,0,0,0] SILT_LIST = [0,0,0,0,0,0,0] CLAY_LIST = [0,0,0,0,0,0,0] BULK_DENSITY_LIST = [0,0,0,0,0,0,0] for i in range(0,len(LIST_TEXTURE)): #print "%d = %s " %(i,LIST_TEXTURE[i]) if (LIST_TEXTURE[i] is None or LIST_TEXTURE[i] == ""): SAND_LIST[i] = 0 SILT_LIST[i] = 0 CLAY_LIST[i] = 0 BULK_DENSITY_LIST[i] = 0 else: record = get_collection_value_from_texture(LIST_TEXTURE[i]) SAND_LIST[i] = float(record[0]) SILT_LIST[i] = float(record[1]) CLAY_LIST[i] = float(record[2]) BULK_DENSITY_LIST[i] = float(record[3]) BIG_LIST = [SAND_LIST,SILT_LIST,CLAY_LIST,BULK_DENSITY_LIST] return BIG_LIST except Exception,err: print err return None
def convert_wtg_to_dly(): try: if (not os.path.exists(WTG_FOLDER)): sys.exit("---ERROR : WTG FOlder does not Existed") WTG_Files = get_files_path(WTG_FOLDER) for f in WTG_Files: wtg_file = os.path.join(WTG_FOLDER, f) print(" ====Process File: %s " % (wtg_file)) name_dly = f[:f.index('.')] if f.endswith(EXTEND_WTG_FILE): name_dly = name_dly + ".DLY" dly_file = os.path.join(DLY_FOLDER,name_dly) with open(wtg_file,'r') as file: data = file.readlines() new_data = [] for i in range(5,len(data)): line = data[i] original_date = line[:5] #original_srad = line[5:11] #fload_orignal_srad = float(orginal_srad) # Process date time date = get_date_string_from_date_in_wtg(original_date) new_line = line[5:].strip() new_line = date + " " + new_line # Process new_line = new_line + " 0.00 0.00 0.000\n" new_data.append(new_line) with open(dly_file,'w') as file: file.writelines(new_data) # file.write("\n") except Exception, err: sys.stderr.write('---[Error]: Write file raised Error %s ' % (err))
def split_n(lis, number): ret_lis = [] for elem in lis: if len(elem[0].split(" ")) == number: ret_lis.append(elem) return ret_lis
def __call__(self, *args, **kwargs): r = self.sendToHub(('EXP_DEVICE', 'DEV_RPC', self.device_class, self.method_name, args, kwargs)) r = r[1:] if len(r) == 1: return r[0] else: return r
def delete_all_pictures(self): """ Deletes all pictures for this plant from the filesystem :return: None """ for i in range(len(self.pictures) - 1, -1, -1): self.delete_picture(i)
def test(): println('GO pyparse.parser.m3u') name = 'M3uParser m3u' try: s = """one_second.mkv empty.mkv """ p = M3uParser(s) m3u = p.rule_M3u() assert m3u.ext is None assert len(m3u.resources) == 2, m3u assert m3u.resources[0].address == 'one_second.mkv', m3u assert m3u.resources[0].ext is None, m3u assert m3u.resources[1].address == 'empty.mkv', m3u assert m3u.resources[1].ext is None, m3u println('PASS', name) except: println('FAIL', name) raise # print traceback name = 'M3uParser extm3u' try: s = """#EXTM3U #EXTINF:1,One Second one_second.mkv #EXTINF:0,Empty empty.mkv """ p = M3uParser(s) m3u = p.rule_M3u() assert m3u.ext == '#EXTM3U', m3u assert len(m3u.resources) == 2, m3u assert m3u.resources[0].address == 'one_second.mkv', m3u assert m3u.resources[0].ext == '#EXTINF:1,One Second', m3u assert m3u.resources[1].address == 'empty.mkv', m3u assert m3u.resources[1].ext == '#EXTINF:0,Empty', m3u println('PASS', name) except: println('FAIL', name) raise # print traceback println('OG pyparse.parser.m3u')
def __init__(self, path): self.path = path # Creating Training -> Training Dev, and Testset d = {} l = 0 self.testsets = {} self.trainingsets = {} print("Number of classes " + str(len(self.trainingsets)) + " number of images " + str(l))
def _join__neg_(lis1, lis_neg): ret = [] done = [] for elem in lis_neg: if len(elem) > 1: ret.append(elem) elem = elem.split(" ") item = [elem[1], elem[2]] done.append(item) for elem in lis1: if len(elem) > 1: if elem not in done: ret.append(elem) return ret
def main(configurationDirectory): import sys if len(sys.argv)>1: configFile=unicode(sys.argv[1]) runtime=ExperimentRuntime(configurationDirectory, configFile) else: runtime=ExperimentRuntime(configurationDirectory, "experiment_config.yaml") runtime.start()