Example #1
0
 def getValidationData(self):
     if self.x_valid is None:
         x_tmp = []
         y_tmp = []
         print("Starting to load validation sets ")
         for c, files in self.trainingsets.iteritems():
             y = self.testsets.keys().index(
                 c)  #We use the testset as a reference
             for file in files:
                 pics = cv2.imread(self.path + c + '/' + file,
                                   cv2.CV_LOAD_IMAGE_GRAYSCALE
                                   )  #unit8 from e.g. 34 to 255
                 x_tmp.append(np.reshape(
                     pics / 255.,
                     len(pics)**2))  #To floats from 0 to 1
                 y_tmp.append(y)
         print("Finished, loading validation sets")
         perm = np.random.permutation(len(y_tmp))
         self.x_valid = theano.shared(np.asarray(
             x_tmp, theano.config.floatX)[perm],
                                      borrow=True)
         self.y_valid = T.cast(
             theano.shared(np.asarray(y_tmp, theano.config.floatX)[perm],
                           borrow=True), 'int32')
     return self.x_valid, self.y_valid
Example #2
0
    def post(self, *args, **kwargs):
        self.adminAuthenticated()

        db = self.getDBSession()

        appId = self.get_argument('appId', '')
        if len(appId):
            self.app = db.query(App).filter(App.appId == appId).first()
            if not self.app:
                raise RuntimeError('Can`t find app for editing')
        else:
            self.app = App()

        name = self.get_argument('name', '')
        code = self.get_argument('code', '')
        errors = []

        if len(name) == 0:
            errors.append(u'Название не должно быть пустое')
        else:
            self.app.name = name

        if len(code) == 0:
            errors.append(u'Код не должен быть пустым')
        else:
            self.app.code = code

        if not len(errors):
            db.merge(self.app)
            db.commit()
            self.redirect('/admin/apps/')
        else:
            self.run()
Example #3
0
def get_top_m_n_grams(m, n_gram):
    """Reduces a list of n_grams with word sequences and their frequencies
    to the top *m* ones.
    
    :param m: amount of sequences and corresponding frequencies to return
    :param n_gram: an object with attribues 'sequencies' and (index-wise) corresponding 'frequencies'
    :returns: The top *m* sequences (and their frequencies) of the input n-gram in a descending order.
    """
    
    top_m_freqs = [0]
    top_m_index = [0]
    for seq in n_gram["sequences"]:
        freq = n_gram["frequencies"][n_gram["sequences"].index(seq)]
        for i in range(0, len(top_m_freqs)):
            if freq > top_m_freqs[i]:
                top_m_freqs.insert(i, freq)
                top_m_index.insert(i, n_gram["sequences"].index(seq))
                if len(top_m_freqs) > m:
                    top_m_freqs.pop()
                    top_m_index.pop()
                break
            
    result = { "sequences": [], "frequencies": [] }
    
    for index in top_m_index:
        result["sequences"].append(n_gram["sequences"][index])
        result["frequencies"].append(n_gram["frequencies"][index])
        
    return result
Example #4
0
 def giveMeNewTraining(self):
     x_tmp = []
     y_tmp = []
     print("Starting to create new training data ")
     for c, files in self.trainingsets.iteritems():
         y = self.testsets.keys().index(
             c)  #We use the testset as a reference
         for file in files:
             image = cv2.imread(
                 self.path + c + '/' + file,
                 cv2.CV_LOAD_IMAGE_GRAYSCALE)  #unit8 from e.g. 34 to 255
             image2 = expandTraining.distorb(image / 255.)
             # cv2.imshow('org', cv2.resize(image, (280, 280)))
             # cv2.imshow('mani', cv2.resize(image2, (280, 280)))
             # cv2.waitKey(2000)
             x_tmp.append(np.reshape(
                 image2,
                 len(image2)**2))  #To floats from 0 to 1
             y_tmp.append(y)
     print("Finished, creating new training data")
     perm = np.random.permutation(len(y_tmp))
     return theano.shared(np.asarray(x_tmp, theano.config.floatX)[perm],
                          borrow=True), T.cast(
                              theano.shared(np.asarray(
                                  y_tmp, theano.config.floatX)[perm],
                                            borrow=True), 'int32')
def rule_one_neg(text, sentence, pos_tagged):
    _dict = {}

    text = []

    for word_tagged in pos_tagged:
        _dict[word_tagged[0]] = word_tagged[1]
        if len(word_tagged[0]) > 1:
            if (word_tagged[0] not in stopwords_set) or (word_tagged[0] in negation):
                text.append(word_tagged[0])

    ret_list = []

    i = 0
    while i < len(text):
        word_set = text[i:i + 3]
        if len(set(word_set).intersection(negation)) > 0 and len(word_set) is 3:
            lis = []
            for word in word_set:
                lis.append((word, _dict[word]))

            if lis[1][1] in adj and lis[2][1] in noun:
                stri = lis[0][0] + " " + lis[1][0] + " " + lis[2][0]
                ret_list.append(stri)

            if lis[0][1] in adj and lis[1][1] in noun:
                stri = lis[2][0] + " " + lis[0][0] + " " + lis[1][0]
                ret_list.append(stri)

        i += 1
    return ret_list
Example #6
0
def compute_most_probable_digraph(so_char_analysis, start):
    """Recursively computes the most probable digraph given a second-order
    correlation matrix as input and a character to start with.
    When the successor of the start character is found all possible
    successors of this character are removed in order to avoid loops."""
    
    characters = so_char_analysis["characters"]
    frequencies = so_char_analysis["frequencies"]
    
    if len(characters) == 0:
        return start
    else:
        current_char = start
        next_char = ""
        next_max_freq = 0
        remove_list = []
        
        for i in range (0, len(characters)):
            if characters[i].startswith(current_char):
                remove_list.append(characters[i])
                if frequencies[i] > next_max_freq:
                    next_max_freq = frequencies[i]
                    next_char = characters[i][1]
        
        for i in range(0, len(remove_list)):
            index = characters.index(remove_list[i])
            characters.remove(characters[index])
            frequencies.remove(frequencies[index])
        
        return start + compute_most_probable_digraph({ "characters": characters, "frequencies": frequencies }, next_char)
Example #7
0
def main():
    # prepare data
    trainingData = []
    testData = []
    split = 0.67

    loadDataset('data/train.csv', split, trainingData, testData)
    # loadDataset('actualdata/train.csv', split, trainingData, testData)

    matches = 0

    for x in range(len(testData)):
        instance = testData[x]
        neighbors = getNeighbors(trainingData,instance,3)
        guess = int(getGuessedDigit(neighbors))
        actual = int(instance[0])

        echo('(%d) Guessed Value %d ' %(x , guess))
        echo('(%d) Actual Value %d ' %(x , actual))

        if(guess == actual):
            matches +=1
        else:
            echo('Displaying character which is not matching')
            # plt.drawPixels(instance[1:], nRows=28, nCols=28)

    echo ('Total matches %d out of %d' %(matches,len(testData)))

    echo ('Accuracy is %f  ' %((matches)/float(len(testData))*100)+'%')
Example #8
0
def ber_calc(init_frame, end_frame):
    error = 0
    for x in range(0, len(init_frame)):
        if init_frame[x] != end_frame[x]:
            error += 1
    ber = (error / len(init_frame))
    return ber
def read_Soil_Data_From_Wise3_Horizon_File(in_csv):
    WISE_HORIZON_RECORD = []
    WISE_HORIZON_DATA_LIST_RECORD = []
    WISE_HORIZON_DATA_LIST_LAYER_BY_RECORD = []
    try:
       f = open(in_csv, "rb")
       l = UnicodeReader(f)
       l.next()
       consider_id = "" 
       for row in l:
           WISE3_ID = str(row[0]).upper()
           WISE3_honu = str(row[1]).upper().strip()
           if (str(row[3]).strip() is not None and str(row[3]).strip() != ""):
                   WISE3_topdep = float(row[3])
           else:
                   WISE3_topdep = float(0)
           if (str(row[4]).strip() is not None and str(row[4]).strip() != ""):
                   WISE3_botdep = float(row[4])
           else:        
                   WISE3_botdep = float(0)
           if (str(row[7]).strip() is not None and str(row[7]).strip() != ""):
                   WISE3_orgc = float(row[7])
           else:
                   WISE3_orgc = float(-9.9)
           if (str(row[23]).strip() is not None and str(row[23]).strip() != ""):
                   WISE3_sand = float(row[23])
           else:
                   WISE3_sand = float(-9.9)
           if (str(row[24]).strip() is not None and str(row[24]).strip() != ""):
                   WISE3_silt = float(row[24])
           else:
                   WISE3_silt = float(-9.9)
           if (str(row[25]).strip() is not None and str(row[25]).strip() != ""):
                   WISE3_clay = float(row[25])
           else:
                   WISE3_clay = float(-9.9)    
            
           if (str(row[27]).strip() is not None and str(row[27]).strip() != ""):
                   WISE3_bulkdens = float(row[27])
           else:
                   WISE3_bulkdens = float(-9.9)
                   
                   
           if (WISE3_ID != consider_id):
               if (WISE_HORIZON_DATA_LIST_LAYER_BY_RECORD is not None and len(WISE_HORIZON_DATA_LIST_LAYER_BY_RECORD) > 0):
                   WISE_HORIZON_DATA_LIST_RECORD.append(WISE_HORIZON_DATA_LIST_LAYER_BY_RECORD)
               WISE_HORIZON_DATA_LIST_LAYER_BY_RECORD = []
               consider_id = WISE3_ID
               WISE_HORIZON_RECORD = [WISE3_ID,WISE3_honu,WISE3_topdep,WISE3_botdep,WISE3_orgc,WISE3_sand,WISE3_silt,WISE3_clay,WISE3_bulkdens]
               WISE_HORIZON_DATA_LIST_LAYER_BY_RECORD.append(WISE_HORIZON_RECORD)
           else:
               WISE_HORIZON_RECORD = [WISE3_ID,WISE3_honu,WISE3_topdep,WISE3_botdep,WISE3_orgc,WISE3_sand,WISE3_silt,WISE3_clay,WISE3_bulkdens]
               WISE_HORIZON_DATA_LIST_LAYER_BY_RECORD.append(WISE_HORIZON_RECORD)
         
       if (WISE_HORIZON_DATA_LIST_LAYER_BY_RECORD is not None and len(WISE_HORIZON_DATA_LIST_LAYER_BY_RECORD) > 0):
           WISE_HORIZON_DATA_LIST_RECORD.append(WISE_HORIZON_DATA_LIST_LAYER_BY_RECORD)
       return  WISE_HORIZON_DATA_LIST_RECORD   
    except Exception, err:
       print err
       return -1
Example #10
0
def merge_sort(A):
    if len(A) <= 1:
        return A
    middle = int(len(A) / 2)
    left = merge_sort(A[:middle])
    right = merge_sort(A[middle:])
    return merge(left, right)
def calculate_Normalization(BIG_LIST,entry_plot_data_numberous):
    print "=== Normalization Calculation====="
    datasets = []
    for i in range(0,len(BIG_LIST)):
        datasets.append(BIG_LIST[i][3])
    #print "Before : "
    #print datasets[0]
    max_list_value = []
    min_list_value = []
    for i in range(0,len(datasets[0])):
         max_list_value.append(-9999999999999999999999999999)
         min_list_value.append(9999999999999999999999999999)
    for record in datasets:
        for i in range(0,len(record)): 
            if (float(record[i]) > max_list_value[i]):
                max_list_value[i] = float(record[i])
            if (float(record[i]) < min_list_value[i]):
                min_list_value[i] = float(record[i])
    
    for i in range(0,len(entry_plot_data_numberous)):
        if (float(entry_plot_data_numberous[i]) > max_list_value[i]):
                max_list_value[i] = float(entry_plot_data_numberous[i])
        if (float(entry_plot_data_numberous[i]) < min_list_value[i]):
                min_list_value[i] = float(entry_plot_data_numberous[i])
        entry_plot_data_numberous[i] = (float(entry_plot_data_numberous[i]) - float(min_list_value[i])) / (float(max_list_value[i]) - float(min_list_value[i]))
                
    print "Max List : " 
    print max_list_value
    print "Min List : " 
    print min_list_value
    for record in datasets:
        for i in range(0,len(record)):
            new_record_value = (float(record[i]) - float(min_list_value[i])) / (float(max_list_value[i]) - float(min_list_value[i]))
            record[i] = new_record_value
Example #12
0
	def test_process_negative_edge(self):
		# outer = 148, left half (first half of data though!)
		xds, yds = self.getInnerAndDetDatasetForGivenOuterValue(148.0)
		x = dnp.asarray(xds.data[:(len(xds) / 2)])
		y = dnp.asarray(yds.data[:(len(yds) / 2)])
		expected = 0.000000,  0.000000, -3.896408,  0.006343,  1.699449,  0.006343
		check(TwoGaussianEdges()._process(x, y), expected, TwoGaussianEdges().labelList)
Example #13
0
	def test_process_positive_edge(self):
		# outer = 148, right half (second half of data though!)
		xds, yds = self.getInnerAndDetDatasetForGivenOuterValue(148.0)
		x = dnp.asarray(xds.data[(len(xds) / 2):])
		y = dnp.asarray(yds.data[(len(yds) / 2):])
		expected = -3.944682,  0.006067,  0.000000,  0.000000,  1.683015,  0.006067
		check(TwoGaussianEdges()._process(x, y), expected, TwoGaussianEdges().labelList)
Example #14
0
def function_to_run(review):
    ret_list = []
    for _, row in review.iterrows():
        _scores_ = {}
        del row['_id']
        tags = row['final']
        if len(set(tags.keys())) > 0:
            for key in tags.keys():
                _scores_[key] = {}
                word = key.split(" ")
                for category in categories.keys():
                    cat = categories[category]
                    try:
                        _scores_[key][category] = sum(
                            word_vectors.n_similarity(word, cat))
                    except Exception as e:
                        break
                        pass

        row['score'] = _scores_
        ret_list.append(row)
        if len(ret_list) > 10000:
            df = pd.DataFrame(ret_list)
            to_mongo_db(df, 'yelp_review_scored_pair_all_not_final')
            print("Written to DB", len(ret_list), 'time from start',
                  (time.time() - start_time))
            ret_list = []

    df = pd.DataFrame(ret_list)
    to_mongo_db(df, 'yelp_review_scored_pair_all_not_final')
    print("Written to DB", len(ret_list), 'time from start',
          (time.time() - start_time))
def merge_sort(A):
    if len(A)<=1:
        return A
    middle = int(len(A)/2)
    left = merge_sort(A[:middle])
    right = merge_sort(A[middle:])
    return merge (left,right)
Example #16
0
    def parse_line(self, line):
        tag = {}
        m = re.search('(?P<indent>[\s\t]*)//@([\s\t]*)(?P<keyword>[a-zA-Z0-9_]+)(?P<info>.*)', line)
        if (m != None):
            keyword = m.group('keyword')
            tag["keyword"] = keyword
            data = {}
            indent = m.group('indent')
            if (indent != None):
                tag['indent'] = len(indent)
            else:
                tag['indent'] = 0

            info = m.group('info')
            while (info and len(info) > 0):
                m = re.search('(?P<key>[a-zA-Z0-9_]+)(=?)(?P<value>[^\s]*)(?P<info>.*)', info)
                if (m != None):
                    key = m.group('key')
                    value = m.group('value')
                    data[key] = value
                    info = m.group('info')
                else:
                    info = None
            tag["data"] = data
            #print tag
        return tag
Example #17
0
def create_reference(sis_to_mean):
    meanpath = settings.referencefile

    for f in sis_to_mean[:]:
        if not f.endswith(".sis"):
            sis_to_mean.remove(f)

    if len(sis_to_mean) > 0:
        tot_K = None
        tot_Na = None

        for file in sis_to_mean:
            K, Na = loadimg(file)
            if tot_K != None:
                tot_K += K
            else:
                tot_K = K
            if tot_Na != None:
                tot_Na += Na
            else:
                tot_Na = Na

        tot_K /= len(sis_to_mean)
        tot_Na /= len(sis_to_mean)

        tot = numpy.concatenate(
            (numpy.ma.filled(tot_K, 0), numpy.ma.filled(tot_Na, 0)))
        tot = (tot + 1) * 6553.6

        write_raw_image(meanpath, tot)
Example #18
0
    def parse_line(self, line):
        tag = {}
        m = re.search(
            '(?P<indent>[\s\t]*)//@([\s\t]*)(?P<keyword>[a-zA-Z0-9_]+)(?P<info>.*)',
            line)
        if (m != None):
            keyword = m.group('keyword')
            tag["keyword"] = keyword
            data = {}
            indent = m.group('indent')
            if (indent != None):
                tag['indent'] = len(indent)
            else:
                tag['indent'] = 0

            info = m.group('info')
            while (info and len(info) > 0):
                m = re.search(
                    '(?P<key>[a-zA-Z0-9_]+)(=?)(?P<value>[^\s]*)(?P<info>.*)',
                    info)
                if (m != None):
                    key = m.group('key')
                    value = m.group('value')
                    data[key] = value
                    info = m.group('info')
                else:
                    info = None
            tag["data"] = data
            #print tag
        return tag
Example #19
0
def test():
    println('GO pyparse.parser.endless_sky')

    name = 'EndlessSkyParser with valid data'
    try:
        s = """# comment
root1 token1 "token 2" `token 3`
    child1.1 token 1.1
    child1.2

root2

# ignored
"""
        p = EndlessSkyParser(s)
        node = p.maybe(p.rule_DataNode)
        assert node is not None, node
        assert node.indent == '', node
        assert node.tokens == (
            'root1',
            'token1',
            '"token 2"',
            '`token 3`',
        ), node
        assert node.lines == slice(1, 5), node  # includes the comment
        assert len(node.children) == 2, node

        assert node.children[0].indent == '    ', node
        assert node.children[0].tokens == (
            'child1.1',
            'token',
            '1.1',
        ), node
        assert node.children[0].lines == slice(3, 4), node
        assert len(node.children[0].children) == 0, node

        assert node.children[1].indent == '    ', node
        assert node.children[1].tokens == ('child1.2', ), node
        assert node.children[1].lines == slice(4, 5), node
        assert len(node.children[1].children) == 0, node

        node = p.maybe(p.rule_DataNode)
        assert node is not None, node
        assert node.indent == '', node
        assert node.tokens == ('root2', ), node
        assert node.lines == slice(5, 7), node  # includes the empty line
        assert len(node.children) == 0, node

        node = p.maybe(p.rule_DataNode)
        assert node is None, node  # ignores the rest
        println('PASS', name)
    except:
        println('FAIL', name)
        raise  # print stacktrace

    # TODO test invalid data
    # TODO test s?

    println('OG pyparse.parser.endless_sky')
Example #20
0
 def handle_data(self, data):
     try:
         if "LoggerAccesibility" in data:
             inicio = data.find("LoggerAccesibility")
             sub = data[inicio + len('LoggerAccesibility') + 1 +
                        len('http://') + 1:]
             fin = sub.find(',true') - 1
             self.server = sub[:fin]
     except:
         ui.message("Error en handle data")
Example #21
0
def readpathfile():
    with open(pathfilename) as f:
        possiblepaths = f.readlines()
    length = len(possiblepaths)
    """splitting strings in sublists and converting to int"""
    for i in range(length):
        possiblepaths[i] = possiblepaths[i].split()
    for i in range(length):
        for j in range(len(possiblepaths[i])):
            possiblepaths[i][j] = int(possiblepaths[i][j])
    return possiblepaths, length
def check_compatible(time,lat,long):
   if (len(time) == 366 or len(time) == 365):
       if (len(long) == 1440 and len(lat) == 720):
           if (lat[0] == 89.875 and lat[719] == -89.875):
               if (long[0] == 0.125 and long[1439] == 359.875):
                   return 1
           else:
               return -1    
       else:
           return -1
   else:
       return -1
Example #23
0
def readfiles():
    """this is the empty content list"""
    """numbers are surrounded by a 'wall' = 99"""
    content = [[99, 99, 99, 99, 99, 99], [99, 0, 0, 0, 0, 99],
               [99, 0, 0, 0, 0, 99], [99, 0, 0, 0, 0, 99],
               [99, 0, 0, 0, 0, 99], [99, 99, 99, 99, 99, 99]]
    """open file and reading file from tesser, skipping empty lines"""
    with open('%s.txt' % (outfilename)) as f_in:
        contenttemp = (line.rstrip() for line in f_in)
        contenttemp = list(line for line in contenttemp
                           if line)  # Non-blank contenttemp in a list
        f_in.close()

    length = len(contenttemp)
    print len(contenttemp[0])
    """splitting every element in more elements, creating a 2D Matrix"""
    for i in range(0, length):
        contenttemp[i] = contenttemp[i].split()
    """raising TypeError if length of contenttemp is larger then 4, if OCR is successful, no Error here"""
    for i in range(0, length):
        if (len(contenttemp[i]) > 4):
            raise TypeError
    """integrating contenttemp to the already created content list"""
    for index in range(4):
        for innerindex in range(4):
            content[index + 1][innerindex + 1] = contenttemp[index][innerindex]
    """converting content to int"""
    for index in range(1, 5):
        for innerindex in range(1, 5):
            content[index][innerindex] = int(content[index][innerindex])
    """raise TypeError if a 0 exists in content, if OCR is successful, no Error here"""
    for x in range(1, 5):
        for y in range(1, 5):
            if (content[x][y] == 0):
                raise TypeError
    """raises TypeError if 'border' is not 99, if OCR is successful, no Error here"""
    for y in range(0, 5):
        if (content[0][y] != 99):
            print "true"
            raise TypeError
    for y in range(0, 6):
        if (content[5][y] != 99):
            print "true"
            raise TypeError
    for x in range(0, 6):
        if (content[x][5] != 99):
            print "true"
            raise TypeError
    for x in range(0, 6):
        if (content[x][5] != 99):
            print "true"
            raise TypeError
    return content
Example #24
0
def initialize_third_order_matrix():
    """Initializes a third-order matrix [2x64000]."""
    
    characters = []
    frequencies = []
    
    for i in range(0, len(alphabet)):
        for j in range(0, len(alphabet)):
            for k in range(0, len(alphabet)):
                characters.append(alphabet[i] + alphabet[j] + alphabet[k])
                frequencies.append(0)
                
    return { "characters": characters, "frequencies": frequencies }
Example #25
0
 def __exit__(self, exc_type, exc_value, traceback):
     r""" Keep or revert parser state. """
     if not isinstance(self, ParserSkeleton):
         raise RuntimeError, type(self)  # expecting ParserSkeleton
     if not len(self.stack) > 0:
         raise RuntimeError, len(self.stack)  # expecting a non-empty stack
     _state = self.stack.pop(-1)
     if exc_type is None and exc_value is None and traceback is None:
         # all ok, forget previous state
         pass
     else:
         # has exception, revert to previous position and propagate exception
         self.state = _state
def main():
    if RECORDER_NAME.upper() == "ALL":
        print "Developed LandInfo Photo"
        if (LAST_DATE is None) or (len(LAST_DATE) <> 8) or not int(LAST_DATE):
            last_update_date = support_rhm_sync_data.get_last_update_date()
            if (last_update_date is None) or (len(last_update_date) <> 8):
                last_update_date = "20150101"
        else:
            last_update_date = LAST_DATE
        print "\n========================================\n"
        print "\n===Start Working====\n"
        print last_update_date
        print "\n========================================\n"
        # last_update_date = "20140102"
        # Request to Nasim get data
        lst_records = requestPlotPhotos.getListPhotoUrls(last_update_date)
        # print last_update_date
        # For loop all result of Nasim to insert our database
        for record in lst_records:
            try:
                print ("===Consider : %s \n" % (last_update_date))
                print record
                print ("\n")
                landinfor_name = str(record[0])
                landinfor_recorder_name = str(record[1])
                landinfor_landscapeNorthPhotoURL = str(record[4])
                landinfor_landscapeEastPhotoURL = str(record[5])
                landinfor_landscapeSouthPhotoURL = str(record[6])
                landinfor_landscapeWestPhotoURL = str(record[7])
                landinfor_soilPitPhotoURL = str(record[2])
                landinfor_soilSamplesPhotoURL = str(record[3])
                result = support_rhm_sync_data.update_photos_landinfo(
                    landinfor_name,
                    landinfor_recorder_name,
                    landinfor_landscapeNorthPhotoURL,
                    landinfor_landscapeEastPhotoURL,
                    landinfor_landscapeSouthPhotoURL,
                    landinfor_landscapeWestPhotoURL,
                    landinfor_soilPitPhotoURL,
                    landinfor_soilSamplesPhotoURL,
                )
                if result == 1:
                    print "\n ===Update successfully==="
                else:
                    print "\n ===Error ----"

            except:
                pass

    else:
        print "Does not support now !"
def merge(left,right):
    result=[]
    while len(left)>0 and len(right)>0:
        if left[0][0]<= right[0][0]:
            result.append(left[0])
            left=left[1:]
        else:
            result.append(right[0])
            right=right[1:]
    if len(left)>0:
        result+=left
    if len(right)>0:
        result+=right
    return result
Example #28
0
def break_on_5(list):
    list = send_index(list)
    start = 0
    res = []
    while start < len(list):
        if start + 5 <= len(list) - 1:
            end = start + 5
            res.append(merge_sort_pivot(list[start:end]))
            start = end
        else:
            end = len(list)
            res.append(merge_sort_pivot(list[start:end]))
            start = end
    return merge_sort_pivot(res)
Example #29
0
def getNeighbors(trainingSet, testInstance, k):
    distances = []
    length = len(testInstance) - 1
    for x in range(len(trainingSet)):
        if(x==0):
            continue
        dist = euclideanDistance(testInstance, trainingSet[x], length)
        # distances.append((trainingSet[x], dist,trainingSet[x][0]))
        distances.append((dist,trainingSet[x]))
    distances.sort(key=operator.itemgetter(0))
    neighbors = []
    for x in range(k):
        neighbors.append(distances[x])
    return neighbors
def break_on_5(list):
    list=send_index(list)
    start =0
    res = []
    while start<len(list):
        if start+5<=len(list)-1:
            end=start+5
            res.append(merge_sort_pivot(list[start:end]))
            start=end
        else:
            end = len(list)
            res.append(merge_sort_pivot(list[start:end]))
            start=end
    return merge_sort_pivot(res)
def find_data_2(data,key):
    try:
        intLength = len(data)
        for i in range(0, intLength-3):
            line = data[i]
            if (line is None or len(line) < 40):
                continue
            line = line[0:40]
            if (checking_key(line, key) == 1):
                return [line,i]
        return None
    except Exception, err:
        print err
        pass
Example #32
0
def merge(left, right):
    result = []
    while len(left) > 0 and len(right) > 0:
        if left[0][0] <= right[0][0]:
            result.append(left[0])
            left = left[1:]
        else:
            result.append(right[0])
            right = right[1:]
    if len(left) > 0:
        result += left
    if len(right) > 0:
        result += right
    return result
 def get_dirsize_Byte(self, dir):
     """Get a directory of the total number of bytes."""
     if len(dir) <= 0:
         print "get a directory of the total number of bytes params error"
         return 0
     for root, dirs, files in os.walk(dir):
         print "dir is ", dirs, files
         try:
             self.allsizebyte += sum(getsize(join(root,name)) for name in files)
             self.file_countnum += len(files)
         except:
             print sys.exc_info()[0],sys.exc_info()[1]
             pass
         print "dirsize is %d filenum is %d", self.allsizebyte, self.file_countnum
         return self.allsizebyte, self.file_countnum 
Example #34
0
def get_number_layers(ID):
    lst_rock_values = get_list_value_of_rock_fragment_to_consider_number_layer(ID)
    length_lst_rock_values = len(lst_rock_values)
    lst_texture_values = get_values_texture_for_soil_horizon_to_consider_number_layers(ID)
    length_lst_texture_values = len(lst_texture_values)
    
    if (( not int("0") in lst_rock_values) or (not int("0") in lst_texture_values)):
        return MAXIMUM_NUMBER_LAYERS
    elif ((lst_rock_values[length_lst_rock_values-1] == 1) or (lst_texture_values[length_lst_texture_values-1] == 1)):
        return MAXIMUM_NUMBER_LAYERS
    else:
        for i in reversed(range(0,MAXIMUM_NUMBER_LAYERS)):
            if ((lst_rock_values[i] == 1) and (lst_rock_values[i+1] == 0)):
                return (i+1)
    return MAXIMUM_NUMBER_LAYERS
Example #35
0
 def _codepoint(self, low, high=None):
     if high is None:
         high = low
     if not low <= high:
         raise ValueError, (
             low,
             high,
         )  # expecting low <= high
     _p = Utf8Parser(self.state.data)
     _codepoint = _p.rule_Codepoint()
     _codepoint = _codepoint.codepoint
     _n = len(self.state.data) - len(_p.state.data)
     return (
         _codepoint,
         self.consume(_n),
     )
Example #36
0
 def token_HEX(self):
     r"""
     hex
         '0' . '9'
         'A' . 'F'
     """
     _c = self[0]
     assert _c is not None
     if _c >= '0' and _c <= '9':
         return self.consume(len(_c))
     if _c >= 'A' and _c <= 'F':
         return self.consume(len(_c))
     assert False, (
         _c,
         self.state,
     )  # not a token_HEX?
Example #37
0
def arange(n, *xs, **kwds):
    if 'dtype' in kwds:
        elt_t = _get_type(kwds['dtype'])
        del kwds['dtype']
    else:
        elt_t = Int64

    assert len(
        kwds) == 0, "Unexpected keyword arguments to 'arange': %s" % kwds
    array_t = make_array_type(elt_t, 1)
    count = __builtin__.len(xs)
    assert 0 <= count <= 2, "Too many args for range: %s" % ((n, ) + tuple(xs))

    if count == 0:
        start = zero_i64
        stop = n
        step = one_i64
    elif count == 1:
        start = n
        stop = xs[0]
        step = one_i64
    else:
        start = n
        stop = xs[0]
        step = xs[1]

    if elt_t != Int64:
        start = Cast(start, type=elt_t)
        stop = Cast(stop, type=elt_t)
        step = Cast(step, type=elt_t)
    return Range(start, stop, step, type=array_t)
Example #38
0
def filter_npmjs_node_modules_path_of(till, dirpath):
    re = ''
    while dirpath != till and __builtin__.len(dirpath) > 0:
        if os.path.basename(dirpath) != 'node_modules':
            re = os.path.join(os.path.basename(dirpath), re)
        dirpath = os.path.dirname(dirpath)
    return re
Example #39
0
 def __call__(self, *args,**kwargs):
     r = self.sendToHub(('EXP_DEVICE','DEV_RPC',self.device_class,self.method_name,args,kwargs))
     r=r[1:]
     if len(r)==1:
         return r[0]
     else:
         return r
Example #40
0
def author_attribution(text):
    """First, computes the second-order matrix (relative frequencies) of the input text.
    Second, computes the sum of the quadratic deviations of the input text against each author in the database."""
    
    text_char_analysis = analyze_text(text, 2, False)
    freqs_text = abs_to_rel_freq(text_char_analysis, 2)
    
    authors = open("res/authors/index").readlines()
    best_match_sum = maxint
    best_match_author = ""
    for author in authors:
        author = author.replace("\n", "")
        current_sum = 0
        author_char_analysis = js.load(open("res/authors/" + author))
        freqs_author = author_char_analysis["frequencies"] 
        for i in range(0, len(freqs_author)):
            current_sum += (freqs_text[i] - freqs_author[i]) ** 2
        
        print "Text matches " + author + " by " + str(current_sum)
        
        if current_sum < best_match_sum:
            best_match_sum = current_sum
            best_match_author = author
            
    return best_match_author
Example #41
0
def compute_n_gram_words(n, text, n_gram=None):
    """Returns *n*-grams of a given *text* source
    
    :param n: the size of the contiguous sequence of words
    :param text: the text source to be analyzed
    :param n_gram: takes an existing n_gram as input, which will be extended
    :returns: an object n_gram: { 'sequences': [*top **m** **n**-grams*], 'frequencies': [*the frequencies of n-grams*] }
    """
    
    if n_gram == None:
        n_gram = { "sequences": [], "frequencies": [] }
    
    if n == 1:
        total_words = 0
        for word in text.split(" "):
            total_words += 1
            for special_char in special_characters:
                word = word.replace(special_char, "")
            word = word.lower()
            if word not in n_gram["sequences"]:
                n_gram["sequences"].append(word)
                n_gram["frequencies"].append(1)
            else:
                n_gram["frequencies"][n_gram["sequences"].index(word)] += 1
        for i in range(0,len(n_gram["frequencies"])):
            n_gram["frequencies"][i] = n_gram["frequencies"][i] / (0.0 + total_words)
    else:
        #TODO
        pass
    
    return n_gram
Example #42
0
def test_arp_table():
    ssh('arp -n')
    read_cp_arp_table()

    debug_print('ARP table from Linux:')
    linux_arp_dict = {}
    fout = open(fname, 'r')
    for line in fout:
        if len(line.strip()) > 0:
            split_line = line.strip().split()
            if split_line[0] != 'Address' and split_line[1] != '(incomplete)':
                net_addr = split_line[0].strip()
                mac_addr = split_line[2].strip().upper()
                debug_print( "net address: %s   , mac address: %s" %(net_addr, mac_addr))
                linux_arp_dict[net_addr] = mac_addr
                if net_addr not in cp_arp_dict:
                    print '*** ERROR: entry %s %s not in CP ARP table' %(net_addr , linux_arp_dict[net_addr])
                    exit(1)
                else:
                    if mac_addr != cp_arp_dict[net_addr]:
                        print '*** ERROR: entry in linux table not equal to CP ARP table. %s != %s' %(mac_addr , cp_arp_dict[net_addr])
                        exit(1)
                        
    for key in cp_arp_dict:
        if key not in linux_arp_dict:
            print '*** ERROR: entry %s %s not in Linux ARP table' %(key,cp_arp_dict[key])
            exit(1)
        else:
            if cp_arp_dict[key] != linux_arp_dict[key]:
                print '*** ERROR: entry not equal in Linux and CP ARP tables. %s != %s' %(cp_arp_dict[key]  , linux_arp_dict[key])
                exit(1)
    
    debug_print("ARP table test OK")
Example #43
0
    def get_first_picture(self):
        """
        Returns a tuple containing the absolute path to the first image and the angle at which it was taken

        :return: A tuple containing of the absolute image path and the angle (path, angle)
        """
        return self.pictures[0] if len(self.pictures) > 0 else None
Example #44
0
def filter_npmjs_node_modules_path_of(till, dirpath):
    re = ''
    while dirpath != till and __builtin__.len(dirpath) > 0:
        if os.path.basename(dirpath) != 'node_modules':
            re = os.path.join(os.path.basename(dirpath), re)
        dirpath = os.path.dirname(dirpath)
    return re
Example #45
0
def create_val_and_train(df, seed, ids, split_rt=0.20):
    """
        Creates two samples (generally used to create
        train and validation samples)

        Parameters
        ----------------------------------------------------
        ids: this is the level of randomization, so if you want to
        randomize countries, rather than cities, you would
        set this to 'countries'

        split_rate: pct of data to assign as validation

        Output
        ----------------------------------------------------
        trn_for_mods (1-split_rate of df), trn_for_val (split_rate of data)
    """
    np.random.seed(seed)
    # Get vector of de-dupped values of ids
    id_dat = pd.DataFrame(df[ids].drop_duplicates())
    # Create random vector to split train val on
    vect_len = len(id_dat.ix[:, 0])
    id_dat["rand_vals"] = np.array(np.random.rand(vect_len, 1))
    df = pd.merge(df, id_dat, on=ids)
    # split data into two dfs
    trn_for_mods = df[df.rand_vals > split_rt]
    trn_for_val = df[df.rand_vals <= split_rt]
    # drop rand_vals
    trn_for_val = trn_for_val.drop("rand_vals", axis=1)
    trn_for_mods = trn_for_mods.drop("rand_vals", axis=1)
    return trn_for_mods, trn_for_val
Example #46
0
    def rule_DataLine(self):
        r"""
        rule_DataLine = token_WHITESPACE? ( token_DATA token_WHITESPACE? )+ rule_EmptyLine
                      ;; the token_WHITESPACE at the start is the indent
        """
        with self:  # revert state on error
            _s = []
            _lineNumber = self.state.lineNumber
            _indent = self.maybe(self.token_WHITESPACE)
            if _indent is None:
                _indent = self.consume(0)
            _s += [_indent]
            _tokens = []
            while True:
                try:
                    _token = self.token_TOKEN()
                    _tokens += [_token]
                    _s += [_token]
                    _whitespace = self.maybe(self.token_WHITESPACE)
                    if _whitespace is not None:
                        _s += [_whitespace]
                except AssertionError:
                    break  # no more tokens
            _empty_line = self.rule_EmptyLine()
            _s += [_empty_line.s]

        assert len(_tokens) > 0  # expecting a token
        _s = tuple(_s)
        return self.DataLine(s=_s,
                             indent=_indent,
                             tokens=_tokens,
                             lineNumber=_lineNumber)
Example #47
0
 def getValidationData(self):
     if self.x_valid is None:
         x_tmp = []
         y_tmp = []
         print("Starting to load validation sets ")
         for c,files in self.trainingsets.iteritems():
             y = self.testsets.keys().index(c) #We use the testset as a reference
             for file in files:
                 pics = cv2.imread(self.path + c + '/' + file, cv2.CV_LOAD_IMAGE_GRAYSCALE) #unit8 from e.g. 34 to 255
                 x_tmp.append(np.reshape(pics / 255., len(pics)**2)) #To floats from 0 to 1
                 y_tmp.append(y)
         print("Finished, loading validation sets")
         perm = np.random.permutation(len(y_tmp))
         self.x_valid = theano.shared(np.asarray(x_tmp, theano.config.floatX)[perm],borrow=True)
         self.y_valid = T.cast(theano.shared(np.asarray(y_tmp, theano.config.floatX)[perm],borrow=True), 'int32')
     return self.x_valid, self.y_valid
Example #48
0
def len_(iterable):
    import __builtin__
    try: return __builtin__.len(iterable)
    except TypeError:
        count = 0
        for i in iterable: count+=1
        return count
Example #49
0
def arange(n, *xs, **kwds):
  if 'dtype' in kwds:
    elt_t = _get_type(kwds['dtype'])
    del kwds['dtype']
  else:
    elt_t = Int64
    
  assert len(kwds) == 0, "Unexpected keyword arguments to 'arange': %s" % kwds
  array_t = make_array_type(elt_t, 1) 
  count = __builtin__.len(xs)
  assert 0 <= count <= 2, "Too many args for range: %s" % ((n,) + tuple(xs))
 
  if count == 0:
    start = zero_i64 
    stop = n 
    step = one_i64 
  elif count == 1:
    start = n 
    stop = xs[0]
    step = one_i64 
  else:
    start = n 
    stop = xs[0]
    step = xs[1]
    
  if elt_t != Int64:
    start = Cast(start, type = elt_t)
    stop = Cast(stop, type = elt_t)
    step = Cast(step, type = elt_t)
  return Range(start, stop, step, type = array_t)
Example #50
0
def get_collection_value_from_all_layer_texture(LIST_TEXTURE):
    try:
        SAND_LIST = [0,0,0,0,0,0,0]
        SILT_LIST = [0,0,0,0,0,0,0]
        CLAY_LIST = [0,0,0,0,0,0,0]
        BULK_DENSITY_LIST = [0,0,0,0,0,0,0]
        for i in range(0,len(LIST_TEXTURE)):
            #print "%d = %s " %(i,LIST_TEXTURE[i])
            if (LIST_TEXTURE[i] is None or LIST_TEXTURE[i] == ""):
                SAND_LIST[i] = 0
                SILT_LIST[i] = 0
                CLAY_LIST[i] = 0
                BULK_DENSITY_LIST[i] = 0
            else:
                record = get_collection_value_from_texture(LIST_TEXTURE[i])
                SAND_LIST[i] = float(record[0])
                SILT_LIST[i] = float(record[1])
                CLAY_LIST[i] = float(record[2]) 
                BULK_DENSITY_LIST[i] = float(record[3])
               
        BIG_LIST = [SAND_LIST,SILT_LIST,CLAY_LIST,BULK_DENSITY_LIST]
        return BIG_LIST
    except Exception,err:
        print err
        return None
def convert_wtg_to_dly():
    try:
        if (not os.path.exists(WTG_FOLDER)):
             sys.exit("---ERROR : WTG FOlder does not Existed")
        WTG_Files = get_files_path(WTG_FOLDER)
        for f in WTG_Files:
            wtg_file = os.path.join(WTG_FOLDER, f)
            print(" ====Process File: %s " % (wtg_file))
            name_dly = f[:f.index('.')]
            if f.endswith(EXTEND_WTG_FILE):
                name_dly = name_dly + ".DLY"
                dly_file = os.path.join(DLY_FOLDER,name_dly)
                with open(wtg_file,'r') as file:
                    data = file.readlines()
                new_data = []
                for i in range(5,len(data)):
                    line = data[i]
                    original_date = line[:5]
                    #original_srad = line[5:11]
                    #fload_orignal_srad = float(orginal_srad)
                    # Process date time
                    date = get_date_string_from_date_in_wtg(original_date)
                    new_line = line[5:].strip()
                    new_line = date + " " + new_line
                    # Process 
                    
                    new_line = new_line + "  0.00  0.00 0.000\n"
                    new_data.append(new_line)
                with open(dly_file,'w') as file:
                    file.writelines(new_data)
                #    file.write("\n") 
       
    except Exception, err:
        sys.stderr.write('---[Error]: Write file raised Error %s ' % (err))
Example #52
0
def split_n(lis, number):
    ret_lis = []
    for elem in lis:
        if len(elem[0].split(" ")) == number:
            ret_lis.append(elem)

    return ret_lis
Example #53
0
 def __call__(self, *args, **kwargs):
     r = self.sendToHub(('EXP_DEVICE', 'DEV_RPC', self.device_class,
                         self.method_name, args, kwargs))
     r = r[1:]
     if len(r) == 1:
         return r[0]
     else:
         return r
Example #54
0
    def delete_all_pictures(self):
        """
        Deletes all pictures for this plant from the filesystem

        :return: None
        """
        for i in range(len(self.pictures) - 1, -1, -1):
            self.delete_picture(i)
Example #55
0
def test():
    println('GO pyparse.parser.m3u')

    name = 'M3uParser m3u'
    try:
        s = """one_second.mkv
empty.mkv
"""
        p = M3uParser(s)
        m3u = p.rule_M3u()
        assert m3u.ext is None
        assert len(m3u.resources) == 2, m3u
        assert m3u.resources[0].address == 'one_second.mkv', m3u
        assert m3u.resources[0].ext is None, m3u
        assert m3u.resources[1].address == 'empty.mkv', m3u
        assert m3u.resources[1].ext is None, m3u
        println('PASS', name)
    except:
        println('FAIL', name)
        raise # print traceback

    name = 'M3uParser extm3u'
    try:
        s = """#EXTM3U

#EXTINF:1,One Second
one_second.mkv

#EXTINF:0,Empty
empty.mkv
"""
        p = M3uParser(s)
        m3u = p.rule_M3u()
        assert m3u.ext == '#EXTM3U', m3u
        assert len(m3u.resources) == 2, m3u
        assert m3u.resources[0].address == 'one_second.mkv', m3u
        assert m3u.resources[0].ext == '#EXTINF:1,One Second', m3u
        assert m3u.resources[1].address == 'empty.mkv', m3u
        assert m3u.resources[1].ext == '#EXTINF:0,Empty', m3u
        println('PASS', name)
    except:
        println('FAIL', name)
        raise # print traceback

    println('OG pyparse.parser.m3u')
Example #56
0
 def __init__(self, path):
     self.path = path
     # Creating Training -> Training Dev, and Testset
     d = {}
     l = 0
     self.testsets = {}
     self.trainingsets = {}
     print("Number of classes " + str(len(self.trainingsets)) +
           " number of images " + str(l))
def _join__neg_(lis1, lis_neg):
    ret = []
    done = []

    for elem in lis_neg:
        if len(elem) > 1:
            ret.append(elem)

        elem = elem.split(" ")
        item = [elem[1], elem[2]]
        done.append(item)

    for elem in lis1:
        if len(elem) > 1:
            if elem not in done:
                ret.append(elem)

    return ret
Example #58
0
File: run.py Project: sckim/ioHub
def main(configurationDirectory):
    import sys
    if len(sys.argv)>1:
        configFile=unicode(sys.argv[1])
        runtime=ExperimentRuntime(configurationDirectory, configFile)
    else:
        runtime=ExperimentRuntime(configurationDirectory, "experiment_config.yaml")

    runtime.start()