Example #1
0
    def do_GET(self):
        GET_parameters = parse_qs(urlparse.urlparse(self.path).query)

        currency_code = GET_parameters.get(CURRENCY_CODE_PARAMETER)
        currency_code = currency_code[0] if currency_code else None

        amount = GET_parameters.get(REQUESTED_AMOUNT_PARAMETER)
        amount = amount[0] if amount else 1

        if currency_code == USD_CODE and isfloat(amount):
            converted_value = currencies_parser.convert_roubles(
                currency_code, float(amount))

            self.send_response(200)
            self._set_headers()
            self._json_response({
                'currency_code': USD_CODE,
                'requested_amount': float(amount),
                'result_amount': converted_value
            })
        else:
            msg = ''
            if currency_code is None:
                msg = 'Not found "currency_code" parameter'
            elif currency_code and currency_code != USD_CODE:
                msg = 'ValueError in "currency_code" parameter.'
            elif not isfloat(amount):
                msg = 'ValueError: "amount" parameter must be a number'

            self.send_response(404)
            self._set_headers()
            self._json_response({'error': msg})
Example #2
0
def main():
        #get file handle
        path = '/home/orcudy/Desktop/cs170a/data/Iris.csv'
        fd = open(path, 'r')
        lines = list(fd)
        fd.close()

        #retrieve flower data
        featureIDs = csv.generateFeatureIDList(lines, ',')

        #remove whitespace from IDs
        for index in range(len(featureIDs)):
                featureIDs[index] = filter(lambda x: x.isalnum(), featureIDs[index])
  
        #generate feature map and convert to appropriate type
        floatConversionUnit = (featureIDs, lambda x: float(x) if utils.isfloat(x) else -1.0)
        dataMap = csv.convertType( csv.generateDataMap(lines, featureIDs), [floatConversionUnit])
        

        #set parameters for analysis
        species = csv.generateArray(dataMap, ['Species']).T
        basePath =  '/home/orcudy/Desktop/cs170a/logs'
        ID = 'iris'
        data = csv.generateArray(dataMap, featureIDs)
        correlation = linalg.computeCorrelationMatrix(data, featureIDs, True, basePath, ID)
        u, s, vt = linalg.computeSVD(correlation, featureIDs, True, basePath, ID)
        renderIrisPCA(u, data, species)
Example #3
0
 def _import_dx(filename):
     origin = delta = data = dims = None
     counter = 0
     with open(filename, 'r') as dxfile:
         for row in dxfile:
             row = row.strip().split()
             if not row:
                 continue
             if row[0] == '#':
                 continue
             elif row[0] == 'origin':
                 origin = np.array(row[1:], dtype=float)
             elif row[0] == 'delta':
                 delta = np.array(row[2:], dtype=float)
             elif row[0] == 'object':
                 if row[1] == '1':
                     dims = np.array(row[-3:], dtype=int)
                     data = np.empty(np.prod(dims))
             elif isfloat(row[0]):
                 data[3 * counter:min(3 *
                                      (counter + 1), len(data))] = np.array(
                                          row, dtype=float)
                 counter += 1
     data = data.reshape(dims)
     return origin, delta, data
Example #4
0
 def classify_headers(headers):
     # first check if we are a list of numbers
     if all(isfloat(x) for x in headers[1:]):
         return HeaderTypes.Data
     # fuzzy check against database
     variety_names = [v.name for v in models.Variety.objects.all()]
     location_names = [l.name for l in models.Location.objects.all()]
     measure_names = models.TrialEntry.measures + tuple(
         measure_aliases.keys())
     vratio = fuzz.token_sort_ratio(headers, ' '.join(variety_names))
     lratio = fuzz.token_sort_ratio(headers, ' '.join(location_names))
     mratio = fuzz.token_sort_ratio(headers, ' '.join(measure_names))
     candidates = (
         (vratio, HeaderTypes.Varieties),
         (lratio, HeaderTypes.Locations),
         (mratio, HeaderTypes.Measures),
     )
     winner = sorted(
         candidates,
         reverse=True,
         key=lambda ratio_type: ratio_type[0],
     )[0]
     ratio, _type = winner
     if ratio < 40:
         _type = None
     return _type
Example #5
0
def parseGetCarriedObjects(ch, argL, cmd):
    """
    Parse and return the items carried by the player, listed in argL.
    Supports:
    cmd item
    cmd i.item
    cmd n item
    cmd all item
    cmd money
    cmd n currency
    cmd all currency
    """

    if utils.isnumber(argL[0]):
	n = int(argL[0])
	i = 1
	if n <= 0:
	    ch.writeToSelf("Heh heh.. sure.. why don't you try that in some alternative universe?\r\n")
	    return 1	    
    elif utils.isfloat(argL[0]):
	n = string.atof(argL[0])
	i = 1
	if n <= 0:
	    ch.writeToSelf("Heh heh.. sure.. why don't you try that in some alternative universe?\r\n")
	    return 1
    elif argL[0] == "all":
	n = -1
	i = 1
    else:
	n = 1
	i = 0


    if len(argL)-1 < i:
	ch.writeToOthers("How many of what do you want to "+cmd+"?\r\n")
	return None


    if n == 1:
	obj = ch.findCarriedObject(argL[i])
	if not obj:
	    ch.writeToSelf("You don't have "+argL[i]+".\r\n")
	    return None

	objL = [obj]
    else:
	objL = ch.findCarriedObjects(argL[i], n)
	if not objL:
	    ch.writeToSelf("You don't have "+argL[i]+".\r\n")
	    return None

	if len(objL) != n and not isinstance(objL[0], Money):
	    ch.writeToSelf("You don't have that many "+argL[i]+".\r\n")
	    return None

    return (objL, i)
 def test_isfloat(self):
     self.assertTrue(utils.isfloat('0'))
     self.assertTrue(utils.isfloat('1'))
     self.assertTrue(utils.isfloat('1.02'))
     self.assertTrue(utils.isfloat('100000'))
     self.assertTrue(utils.isfloat('-1'))
     self.assertFalse(utils.isfloat('lorem'))
     self.assertFalse(utils.isfloat('2t2'))
Example #7
0
    def summary(self,*vars):
        """Display a summary of all recorded notes, checks, records."""

        if vars:
            print("Values Used:")
            print("============")
            print()
            show(*vars,depth=1)
            
        var = self.var
        hd = 'Summary of'
        if var is not None:
            hd += ' '+var+' for '
        hd += self.__class__.__name__
        if self.title:
            hd += ': '+str(self.title)
        
        print()
        print(hd)
        print('=' * len(hd))
        print()
        if self._notes:
            print('Notes:')
            print('------')
            for txt in self._notes:
                print('    -',txt)
            print()
            
        if self._checks:
            print('Checks:')
            print('-------')
            width = max([len(l) for f,l,v,d in self._checks])
            for chk in self._checks:
                print(self.fmt_check(chk,width=width+2))
            print()
                
        hd = 'Values'
        if self.var:
            hd += ' of '+self.var
        hd += ':'
        print(hd)
        print('-'*len(hd))
        width = max([len(l) for l,v,d in self._record])
        
        govval = None
        if var:
            govval = self.selector([d[var] for l,v,d in self._record])
        for rec in self._record:
            print(self.fmt_record(rec,var=var,width=width+1,govval=govval,nsigfigs=self.nsigfigs))

        if govval is not None:
            print()
            h = 'Governing Value:'
            print('   ',h)
            print('   ','-'*len(h))
            print('      ','{0} = {1}'.format(var,(sfrounds(govval,self.nsigfigs) if isfloat(govval) else "{0!r}".format(govval))), self.units if self.units is not None else '')
Example #8
0
    def on_main_button_click(self, sender, sender_name):
        if sender_name != 'OK':
            self.destroy()
            return

        if not( isfloat(self.res['ms1']) and isfloat(self.res['ms2']) ):
            return

        ms1 = int(self.res['ms1'])
        ms2 = int(self.res['ms2'])

        # Create list of subtitles to apply changes
        applyItems = []
        if self.res['applyToSubs'] == 'all lines':
            for item in self.subtitleModel.get_model():
                applyItems.append(item[0])
        else:
            self.get_tv_selection()
            if len(self.tvSelectionList) == 0:
                self.destroy()
                return
            applyItems = self.tvSelectionList[:]

        A0 = int(applyItems[0].startTime)
        B0 = int(applyItems[-1].startTime)
        A = int(applyItems[0].startTime) + ms1 * (1 if self.res['op1'] == 'Add' else -1)
        B = int(applyItems[-1].startTime) + ms2 * (1 if self.res['op2'] == 'Add' else -1)

        if A0 == B0:
            return
            
        for item in applyItems:
            duration = int(item.duration)
            new_start_time = int( grad(A0, B0, A, B, int(item.startTime)) )
            new_stop_time = int(new_start_time) + duration
            self.changeList.append( (item, int(item.startTime), int(item.stopTime), int(new_start_time), int(new_stop_time)) )
            item.startTime = new_start_time
            item.stopTime = new_stop_time

        self.destroy()
Example #9
0
    def _validate(value):
        """
        Ensures that a candidate token contains only valid characters and that
        a single token contains only letters or only numbers

        :param value: string that will be the value of the token
        """
        if LEGAL_CHARS.match(value) is None:
            raise ValueError('Contains illegal characters')

        # Must either be a float or alpha chars
        if not isfloat(value) and not value.isalpha():
            raise ValueError('Token contains letters and numbers')
Example #10
0
 def load(filepath):
     attrs = {}
     with open(filepath, "r") as f:
         for line in f:
             s = [i.strip() for i in line.split(":")]
             if s[1] == "None":
                 attrs[s[0]] = None
             elif utils.isfloat(s[1]):
                 attrs[s[0]] = float(s[1])
             elif s[1].isdigit():
                 attrs[s[0]] = int(s[1])
             else:
                 attrs[s[0]] = s[1]
     return Dict(**attrs)
Example #11
0
def findoutliers(database):
    spark = SparkSession.builder.appName(
        "Python Spark SQL basic example").config("spark.some.config.option",
                                                 "some-value").getOrCreate()
    sc = spark.sparkContext
    sc.addFile("utils.py")
    sc.addFile("onumeric.py")
    sc.addFile("ostring.py")
    lines = sc.textFile(database)
    headers = lines.first()
    rows = lines
    #rows = lines.filter(lambda x: x!=headers)
    headers = headers.split("\t")
    rows = rows.map(lambda l: l.split("\t"))
    rowsdf = spark.createDataFrame(rows)

    outlierslist = []
    for i in sc.range(len(headers)).collect():
        header = headers[i]
        valuelistrdd = rowsdf.select("_" + str(i + 1)).rdd.flatMap(lambda x: x)
        valuelist = valuelistrdd.collect()
        valuelist = valuelist[1:]
        rand_smpl = [
            valuelist[i]
            for i in sorted(random.sample(range(len(valuelist)), 10))
        ]
        smpl_type = utils.gettype(rand_smpl)
        if smpl_type == 'Numeric':
            valuelistnew = onull.find_outliers(valuelistrdd)
            valuelistrdd = valuelistrdd.filter(lambda x: utils.isfloat(x))
            valuelistnew = valuelistnew + onumeric.find_outliers(valuelistrdd)
        elif smpl_type == 'String':
            valuelistnew = []
            valuelistnew = ostring.find_outliers(valuelistrdd)
        elif smpl_type == 'Date':
            valuelistnew = onull.find_outliers(valuelistrdd)
            valuelistrdd = valuelistrdd.filter(lambda x: utils.isdate(x))
            valuelistnew = valuelistnew + odate.find_outliers(valuelistrdd)
        elif smpl_type == 'None':
            valuelistnew = []
        outlierslist.append((header, valuelistnew))

    database = sc.parallelize(outlierslist)

    database.map(lambda x: "{0} \t {1}".format(x[0], x[1])).saveAsTextFile(
        "x.out")
Example #12
0
def parse_dirty_dataset(path):
    rows = []
    with open(path) as csvfile:
        for row in csv.DictReader(csvfile):
            record = {}
            for field_name, value in row.items():
                if field_name in field_map:
                    value = value.strip().replace(',', '.')
                    if utils.isfloat(value):
                        value = float(value)
                    elif value == '-' or value == '':
                        value = make_average_value(field_name)
                elif field_name in text_fields:
                    pass
                else:
                    raise Exception("unknown field '%s'" % repr(field_name))
                record[field_name] = value
            rows.append(record)
    return rows
Example #13
0
    def keywords(self):
        # Creo un dizionario contenente tutte le parole che non sono stopwords
        # in forma lemmatizzata
        # chiave = parola lemmatizzata
        # valore = elenco di istanze di tutte le parole che hanno la stessa lemmatizzazione
        stopw = self.parent.parent.stopw
        words = [
            wd for wd in self.words if not wd.lemmatized in stopw
            and wd.text not in string.punctuation and not isfloat(wd.text)
        ]
        word_dict = {}

        for word in words:
            if not word.lemmatized in word_dict.keys():
                word_dict[word.lemmatized] = [word]
            else:
                word_dict[word.lemmatized].append(word)

        return word_dict
Example #14
0
def main():
        #get file handle
        path = '/home/orcudy/Desktop/cs170a/data/CSV.csv'
        fd = open(path, 'r')
        lines = list(fd)
        fd.close()

        #cleanup first and last question IDs (others are already clean)
        lines[0] = lines[0].split(',')
        for index in [0,-1]:
                lines[0][index] = filter(lambda x: x.isalnum(), lines[0][index])
        lines[0] = ','.join(lines[0])

        #retrieve question IDs
        questionIDs =  csv.generateFeatureIDList(lines, ',')

        #data dynamics
        chars = ['usr']
        floats = ['weight', 'standwt']
        ints = filter(lambda x: x not in chars + floats, questionIDs)
        
        #convert feature map to appropriate type
        floatConversionUnit = (floats, lambda x: float(x) if utils.isfloat(x) else -1.0)
        intConversionUnit = (ints, lambda x: int(x) if x.isdigit() else -1)
        responseMap = csv.convertType( csv.generateDataMap(lines, questionIDs), [floatConversionUnit, intConversionUnit])

        #mask "Don't know" and "Refused" responses
        maskValues = [-1, 8, 9, 98, 99, 998, 999, 9998, 9999, 99999]
        position = csv.generateArray(responseMap, keys.position).T
        basePath = '/home/orcudy/Desktop/cs170a/logs'

        #begin analysis
        datakeys = [keys.demographic, keys.phone, keys.control, keys.internet, keys.social, keys.government, keys.exclusion]
        IDs = ['PCA on Demographic Category', 'PCA on Phone Category', 'PCA on Control Category', 'PCA on Internet category', 'PCA on Social Category', 'PCA on Government Category', 'PCA on Exclusion Category']
        for index in range(len(datakeys)):
                key = datakeys[index]
                ID =  IDs[index]
                data = csv.generateMaskedArray(responseMap, key, maskValues)
                correlation = linalg.computeCorrelationMatrix(data, keys=key, log=True, path=basePath, keyname=ID)
                u, s, vt = linalg.computeSVD(correlation, keys=key, log=True, path=basePath, keyname=ID)
                renderPCA(u, data, position, ID)
Example #15
0
    def check_orthography(self, sub):
        if not self.dict:
            return

        wordList = re.sub("[\W]", " ", remove_tags(sub.text),
                          flags=re.UNICODE).split()
        errorList = [
            word for word in wordList
            if (not self.dict.check(word)) and (not isfloat(word))
        ]
        errorCoord = []
        for word in errorList:
            coords = find_all_str(sub.text, word)
            errorCoord += [coord for coord in coords]

        if len(errorList) > 0:
            sub.info = ('Text-Error-Orthography',
                        (u'Ορθογραφία: ' + ','.join(w for w in errorList),
                         errorCoord))
        else:
            sub.info = ('Text-Error-Orthography', '')
Example #16
0
 def fmt_record(self,rec,width=None,var=None,govval=None,nsigfigs=4):
     """Format a computation record for display."""
     label,_varlist,_vars = rec
     _vars = _vars.copy()
     if width is None:
         width = len(label)
     if var is None:
         var = self.var
     ans = "    {label:<{width}} ".format(label=label+':',width=width)
     if var:
         val = _vars.pop(var)
         ##print(val, type(val))
         ans += '{0} = {1}'.format(var,(sfrounds(val,nsigfigs) if isfloat(val) else "{0!r}".format(val)))
         if self.units:
             ans += ' '+str(self.units)
         if govval is not None:
             if val == govval:
                 ans += '  <-- governs'
     if _vars:
         ans += '\n       ('+fmt_dict(_vars)+')'
     return ans
 def atom(self, tokens):
     token = tokens.pop()
     node = None
     if isfloat(token.value):
         node = Number(token.value)
     elif self.match(token, (TokenType.TRUE, TokenType.FALSE)):
         node = Boolean(token.value)
     elif self.match(token, (TokenType.EMPTY, )):
         node = Empty()
     elif self.match(token, (TokenType.CONST, )):
         node = String(token.value)
     elif self.match(token, (TokenType.IDENT, )):
         if self.match(tokens.peek(),
                       (TokenType.STORE, TokenType.OPEN_PAREN)):
             node = Name(token.value, Context.STORE)
         elif not self.match(
                 tokens.peek(),
             (TokenType.OPEN_PAREN, )):  # not a function dec
             node = Name(token.value, Context.LOAD)
     if node: self.callstack.push(token)
     return node
 def classify_headers(headers):
     # first check if we are a list of numbers
     if all(isfloat(x) for x in headers[1:]):
         return HeaderTypes.Data
         # fuzzy check against database
     variety_names = [v.name for v in models.Variety.objects.all()]
     location_names = [l.name for l in models.Location.objects.all()]
     measure_names = models.TrialEntry.measures + tuple(measure_aliases.keys())
     vratio = fuzz.token_sort_ratio(headers, " ".join(variety_names))
     lratio = fuzz.token_sort_ratio(headers, " ".join(location_names))
     mratio = fuzz.token_sort_ratio(headers, " ".join(measure_names))
     candidates = (
         (vratio, HeaderTypes.Varieties),
         (lratio, HeaderTypes.Locations),
         (mratio, HeaderTypes.Measures),
     )
     winner = sorted(candidates, reverse=True, key=lambda ratio_type: ratio_type[0])[0]
     ratio, _type = winner
     if ratio < 40:
         _type = None
     return _type
Example #19
0
    def on_main_button_click(self, sender, sender_name):
        if sender_name != 'OK':
            self.destroy()
            return

        if self.res['applyToSubs'] is None or self.res['milliseconds1'] is None or self.res['operation1'] is None or self.res['applyToTime1'] is None:
            self.destroy()
            return

        if self.res['applyToTime1'] == 'Start Time' and (self.res['operation2'] is not None and not isfloat(self.res['milliseconds2'])):
            self.destroy()
            return

        if not isfloat(self.res['milliseconds1']):
            self.destroy()
            return

        # Create list of subtitles to apply changes
        applyItems = []
        if self.res['applyToSubs'] == 'all lines':
            for item in self.subtitleModel.get_model():
                applyItems.append(item[0])
        else:
            self.get_tv_selection()
            if len(self.tvSelectionList) == 0:
                self.destroy()
                return
            applyItems = self.tvSelectionList[:]

        if self.res['milliseconds1'] is not None:
            milliseconds1 = int(self.res['milliseconds1']) * (1 if self.res['operation1'] == 'add' else -1)
        else:
            milliseconds1 = 0
        if self.res['milliseconds2'] is not None:
            milliseconds2 = int(self.res['milliseconds2']) * (1 if self.res['operation1'] == 'add' else -1)
        else:
            milliseconds2 = 0

        operation1Items = applyItems[:]
        if milliseconds1 > 0:
            operation1Items.reverse()

        for item in applyItems:
            newBound = [None, None]
            if self.res['applyToTime1'] == 'Start Time':
                newBound[0] = int(item.startTime) + milliseconds1
                newBound[1] = int(item.stopTime) + milliseconds2
            elif self.res['applyToTime1'] == 'Stop Time':
                newBound[0] = int(item.startTime)
                newBound[1] = int(item.stopTime) + milliseconds1
            elif self.res['applyToTime1'] == 'Both Start and Stop Times':
                newBound[0] = int(item.startTime) + milliseconds1
                newBound[1] = int(item.stopTime) + milliseconds1
            else:
                continue

            nextSub = self.subtitleModel.get_next(item)
            prevSub = self.subtitleModel.get_prev(item)

            if newBound[0] < 0:
                return
            if nextSub is not None and newBound[1] > nextSub.startTime - 120:
                continue
            if prevSub is not None and newBound[0] < prevSub.stopTime + 120:
                continue
            if newBound[0] > newBound[1] or newBound[1] - newBound[0] < 1000:
                continue
            self.changeList.append((item, int(item.startTime), int(item.stopTime), int(newBound[0]), int(newBound[1])))
            item.startTime = int(newBound[0])
            item.stopTime = int(newBound[1])
        self.destroy()
Example #20
0
    def on_main_button_click(self, sender, sender_name):
        if sender_name != 'OK' or any(entry is None for entry in self.res.itervalues()) or self.subtitleModel.is_empty() or not(isfloat(self.res['originalMS']) or isfloat(self.res['targetMS'])):
            self.destroy()
            return

        # Create list of subtitles to apply changes
        applyItems = []
        if self.res['applyTo'] == 'all lines':
            for item in self.subtitleModel.get_model():
                applyItems.append(item[0])
        else:
            self.get_tv_selection()
            if len(self.tvSelectionList) == 0:
                self.destroy()
                return
            applyItems = self.tvSelectionList[:]

        origMS = int(self.res['originalMS'])
        targetMS = int(self.res['targetMS'])

        for item in applyItems:
            if (self.res['condition'] == '<' and not(item.duration < origMS)) or (self.res['condition'] == '<=' and not(item.duration < origMS)) or (self.res['condition'] == '>' and not(item.duration > origMS)) or (self.res['condition'] == '>=' and not(item.duration >= origMS)) or (self.res['condition'] == '=' and not(item.duration == origMS)):
                continue

            # Check if new duration is less than the minimum allowed
            minDuration = int ( (20.0 * (item.calc_target_duration() - 500) / 26.9999999999) + 500 ) + 1
            targetDuration = item.duration + targetMS * (1 if self.res['operation'] == 'add' else -1)
            targetDuration = targetDuration if targetDuration >= minDuration else minDuration
            targetDuration = targetDuration if targetDuration >= 1000 else 1000

            # Check if gap is not violated
            newStop = int(item.stopTime) + (int(targetDuration) - int(item.duration))
            nextSub = self.subtitleModel.get_next(item)
            if nextSub is not None and newStop > nextSub.startTime - 120:
                newStop = nextSub.startTime - 120

            # Check if change in stopTime violates min duration
            if int(newStop) - int(item.startTime) < int(minDuration):
                continue

            # Finally Apply newStop
            self.changeList.append( (item, int(item.startTime), int(item.stopTime), int(item.startTime), int(newStop)) )
            item.stopTime = int(newStop)

        self.destroy()
Example #21
0
def readData(class_name,
             class_questionnaire='Q92510',
             data_path=None,
             missing_input='none',
             dummy=False,
             transform_numeric=False,
             use_text=False,
             skip_class_questionnaire=True):
    # attributes are separated by commas (',')
    # "nan" is assigned to fields with 'N/A' or 'None'

    print('Reading data...')
    data = pd.read_csv(data_path,
                       header=0,
                       delimiter=",",
                       na_values=['N/A', 'None', 'nan', 'NAAI', 'NINA'],
                       quoting=0,
                       encoding='utf8')

    if (not transform_numeric):
        dummy = False

    #print(data.columns[-1])
    # data = data.dropna(subset=[class_name])
    # data = data.drop(np.where([e == 'NAAI' or e == 'NINA' for e in data[data.columns[-1]]])[0])
    # print(data.shape)
    data = data.drop(data.columns[data.columns.str.endswith('id')], 1)
    data = data.drop(data.columns[data.columns.str.endswith('token')], 1)
    data = (data.drop(data.columns[data.columns.str.endswith('ipaddr')], 1))
    data = (data.drop(data.columns[data.columns.str.endswith('date')], 1))
    data = (data.drop(data.columns[data.columns.str.endswith('stamp')], 1))
    #data = (data.drop(data.columns[data.columns.str.endswith('participant_code')],1))
    data = (data.drop(data.columns[data.columns.str.endswith('datLesao')], 1))
    data = (data.drop(data.columns[data.columns.str.endswith('datNasc')], 1))
    # data = (data.drop(data.columns[data.columns.str.endswith('Origem')],1))
    # data = (data.drop(data.columns[data.columns.str.endswith('Cidade')],1))
    # data = (data.drop(data.columns[data.columns.str.endswith('Estado')],1))
    # data = (data.drop(data.columns[data.columns.str.endswith('País')],1))
    # data = (data.drop(data.columns[data.columns.str.endswith('participant_code')],1))
    # data = (data.drop(data.columns[data.columns.str.endswith('Natural de')],1))

    #data = ((((data.T).drop_duplicates(keep='first')).dropna(how='all')).T)
    #dropping columns that are constant
    #data = data.loc[:,data.apply(pd.Series.nunique) != 1]

    ## data = pp.preprocess(data_path,class_name)
    n_samples = data.shape[0]
    n_features = data.shape[1]
    regex_date = re.compile(
        '(\d{4})-(\d{2})-(\d{2})\s?((\d{2}):(\d{2}):(\d{2}))?')

    treatment = np.empty(n_features, dtype='U5')

    attributes = []
    categories = []
    transformedData = []
    index = 0
    si = 0

    print('Transforming data...')
    ### representing the categories with numbers

    for attribute in data.columns:

        if skip_class_questionnaire and class_questionnaire is not None and class_questionnaire in attribute and class_name not in attribute:
            index += 1
            continue
        # else:
        # 	if class_questionnaire in attribute and 'Ombro' not in attribute:
        # 		index+=1
        # 		continue

        t = pd.factorize(data[attribute].values, sort=True)

        #temp = t[0]

        i = utils.firstNotNan(data[attribute].values)

        try:
            result = regex_date.match(data[attribute].values[i])
            if (result):
                treatment[index] = 'date'

            elif (len(t[1]) > 0.9 * n_samples and len(t[1][0]) > 50):
                # if(attribute == 'participant_code'):
                # 	temp = t[0]
                # 	treatment[index] = 'int'
                # else:
                treatment[index] = 'text'
            else:
                if (utils.isfloat(data[attribute].values[i])):
                    # index+=1
                    # continue
                    #temp = [float(x) for x in t[0]]
                    treatment[index] = 'float'

                elif (not dummy):

                    if (transform_numeric
                            or utils.isint(data[attribute].values[i])):
                        temp = t[0]
                        # if not utils.isint(data[attribute].values[i]):
                        # 	index += 1
                        # 	continue

                    else:
                        temp = data[attribute].values
                    treatment[index] = 'int'

                else:
                    treatment[index] = 'bin'

        except TypeError:

            if (utils.isfloat(data[attribute].values[i])):
                # index+=1
                # continue
                temp = np.array(data[attribute].values).reshape(-1, 1)
                treatment[index] = 'float'
            elif (utils.isint(data[attribute][i])):
                # index+=1
                # continue
                temp = (np.array(data[attribute].values) * 1).reshape(-1, 1)
                treatment[index] = 'int'
            else:
                print("could not identify type of attribute %s" % attribute)
                exit(-1)

        #treatment of class	attribute
        if (class_name in attribute):
            temp = t[0]
            treatment[index] = 'int'

        if (treatment[index] == 'float'):
            if (missing_input != 'none'):
                imp = preprocessing.Imputer(strategy=missing_input, axis=0)
                temp = imp.fit_transform(
                    X=np.array(data[attribute].values).reshape(-1, 1))
            else:
                temp = data[attribute].values
            #print(np.array(list((float(x) for x in temp))).reshape(-1,1).shape)
            transformedData.append(
                np.array(list((float(x) for x in temp))).reshape(-1, 1))

        else:
            # t[0] corresponds to the translated numeric data
            # t[1] corresponds to a list with the possible values for each feature'
            # (different values in a column, e.g. [sim, não]).
            # the index of that value in the list corresponds to its numeric representation
            # (e.g. [sim, não] -> sim is represented by 0 and não by 1).
            # if(missing_input != 'none' and treatment[index] != 'bin'):
            # 	imp = preprocessing.Imputer(missing_values=-1,strategy=missing_input,axis=0)
            # 	temp = imp.fit_transform(X=temp.reshape(-1,1))

            if (treatment[index] == 'bin'):
                #imp = preprocessing.Imputer(missing_values=-1,strategy='mean',axis=0)
                #temp = imp.fit_transform(X=np.array(temp).reshape(-1,1))

                temp = pd.get_dummies(np.ravel(data[attribute].values))
                for x in temp.columns:
                    attributes.append(attribute + '=' + x)
                    #print(temp[x].reshape(-1,1).shape)
                    transformedData.append(temp[x].reshape(-1, 1))

            elif (treatment[index] == 'int'):
                if (not transform_numeric):
                    temp = data[attribute].values
                    for temp_index in range(len(temp)):
                        if (isinstance(temp[temp_index], str)):
                            temp[temp_index] = temp[temp_index].upper()

                    i = utils.firstNotNan(data[attribute].values)
                    if (utils.isint(data[attribute].values[i])
                            and missing_input != 'none'):
                        temp[data[attribute].values == 'NAAI'] = -1
                        temp[np.isnan(
                            np.array(data[attribute].values,
                                     dtype=float))] = -1
                        imp = preprocessing.Imputer(missing_values=-1,
                                                    strategy=missing_input,
                                                    axis=0)
                        temp = imp.fit_transform(
                            X=np.array(list(int(x)
                                            for x in temp)).reshape(-1, 1))

                elif (missing_input != 'none'):
                    imp = preprocessing.Imputer(missing_values=np.nan,
                                                strategy=missing_input,
                                                axis=0)
                    temp = imp.fit_transform(X=np.array(temp).reshape(-1, 1))

                #print(np.array(temp).reshape(-1,1).shape)
                transformedData.append(np.array(temp).reshape(-1, 1))

            elif (treatment[index] == 'date'):
                temp = []
                for date in data[attribute].values:
                    if (not isinstance(date, float)):
                        temp.append(int(date[:4]))
                    else:
                        temp.append(-1)
                if (missing_input != 'none'):
                    imp = preprocessing.Imputer(strategy='most_frequent',
                                                axis=0)

                    temp = imp.fit_transform(X=np.array(temp).reshape(-1, 1))

                #print(np.array(temp).reshape(-1,1).shape)
                transformedData.append(np.array(temp).reshape(-1, 1))

            elif (use_text and treatment[index] == 'text'):
                #try:
                bigword = ''
                #print(attribute)
                try:
                    bag_of_words = CountVectorizer(min_df=0.25,
                                                   stop_words=sw,
                                                   ngram_range=(1, 4))
                    #print(data[attribute])
                    words = np.array(
                        bag_of_words.fit_transform(
                            ((data[attribute].values))).todense())
                    c = 0
                    for word in bag_of_words.get_feature_names():
                        bigword = bigword + word + ' '
                        attributes.append(attribute + ' termo: ' + word)
                        transformedData.append(words[:, c].reshape(-1, 1))
                        c += 1
                        # wordcloud = WordCloud(stopwords=sw,background_color='white').generate(bigword,)
                        # plt.imshow(wordcloud)
                        # plt.axis('off')
                        # plt.show()
                except (ValueError, AttributeError):

                    index += 1
                    continue
            else:
                index += 1
                continue
            # else:
            # 	print('undefined option for pre processing: (%s, %s) ' % (categ) )
            # 	exit(-1)

        categories.append(t[1])
        if (treatment[index] != 'text' and treatment[index] != 'bin'):
            attributes.append(attribute)

        index += 1

    data = np.array(transformedData).reshape(-1, n_samples).T
    data = pd.DataFrame(data, columns=attributes)

    # pd.DataFrame(data,columns=attributes).to_csv('out.csv', index=False)
    # f = open('DorR.csv', 'w')
    # f.write(','.join(np.array(attributes, dtype=object)))

    # for d in data:
    # 	f.write('\n')
    # 	f.write(','.join(str(dd) for dd in d))
    # exit()

    return data
 def write_database(self):
     # create SignificanceEntry
     significance_entries = []
     name_sigentry = {}
     statistics = self.progress["statistics"]
     # TODO: BUG: if a location has no entries, but has stat entries,
     # TODO: it will not show up in the preview but will be written here
     # TODO: (kinda harmless though, since it won't have any trials mapped to it)
     for name in statistics:
         if not statistics[name]:
             continue
         fieldname = statistics[name]["fieldname"]
         alpha = statistics[name]["alpha"]
         if not alpha:
             alpha = None
         comparing = statistics[name]["comparing"]
         try:
             cells = self.progress["cells"][name]
         except KeyError:
             cells = []
         for header, value in cells:
             if not value or not isfloat(value):
                 continue
             sigentry = models.SignificanceEntry(comparing=comparing, method=fieldname, alpha=alpha, value=value)
             significance_entries.append(sigentry)
             try:
                 name_sigentry[header].append(sigentry)
             except KeyError:
                 name_sigentry[header] = [sigentry]
     for sigentry in significance_entries:
         sigentry.save()
         # create Date
     year = int(self.progress["date"])
     date_pd = datetime.date(year=year, month=5, day=1)
     date_hd = datetime.date(year=year, month=8, day=1)
     try:
         plant_date = models.Date.objects.get(date=date_pd)
     except:
         plant_date = models.Date(date=date_pd)
         plant_date.save()
     try:
         harvest_date = models.Date.objects.get(date=date_hd)
     except:
         harvest_date = models.Date(date=date_hd)
         harvest_date.save()
         # create/update TrialEntry objects
     entries = []
     lpvh_entries = {}
     location_names = []
     variety_names = []
     planting_methods = []
     for trial in models.TrialEntry.objects.filter(harvest_date=harvest_date):
         lpvh = (trial.location.pk, trial.planting_method_tags.pk, trial.variety.pk, trial.harvest_date.pk)
         lpvh_entries[lpvh] = trial
     for entry in self.progress["trial_entries"]:
         location = entry["location"]["pk"]
         planting_methods = entry["location"]["planting_methods"]
         variety = entry["variety"]["pk"]
         lpvh = (location, planting_methods, variety, harvest_date.pk)
         try:
             trial = lpvh_entries[lpvh]
         except KeyError:
             trial = models.TrialEntry(
                 location_id=location,
                 variety_id=variety,
                 planting_method_tags_id=planting_methods,
                 harvest_date=harvest_date,
                 plant_date=plant_date,
             )
             lpvh_entries[lpvh] = trial
         measure = entry["measure"]["fieldname"]
         value = entry["measure"]["value"]
         if value is None:
             continue
         setattr(trial, measure, value)
         entries.append(trial)
         trial.save()
         # grab info for a later requery of these trials
         location_names.append(trial.location.name)
         variety_names.append(trial.variety.name)
         planting_methods.append(trial.planting_method_tags.planting_method)
         # attach TrialEntry objects to SignificanceEntry objects
     for name, sigentries in name_sigentry.items():
         # name is a user-entered string, lookup in our mappings
         # TODO: assume name is a location for now
         location = self.progress["locations"][name]["pk"]
         planting_methods = self.progress["locations"][name]["planting_methods"]
         # TODO: avoid the following database call
         trials = list(
             models.TrialEntry.objects.filter(location_id=location)
             .filter(planting_method_tags_id=planting_methods)
             .filter(harvest_date=harvest_date)
         )
         for sigentry in sigentries:
             sigentry.trials.add(*trials)
             sigentry.save()
             # save TrialEntry query for later editing
     self.progress["submitted"]["locations"] = list(set(location_names))
     self.progress["submitted"]["varieties"] = list(set(variety_names))
     self.progress["submitted"]["planting_methods"] = list(set(planting_methods))
     self.progress["submitted"]["harvest_date"] = {
         "year": harvest_date.date.year,
         "month": harvest_date.date.month,
         "day": harvest_date.date.day,
     }
     self.save()
     self.model.submitted = True
     self.model.save()
    def prepare_table(self):
        ## collect entered data
        trial_entries = {}
        summary_entries = {}
        locations = []
        varieties = []
        measures = []
        # locations, varieties, measures
        lnames = self.progress["headers"][HeaderTypes.Locations]
        vnames = self.progress["headers"][HeaderTypes.Varieties]
        mnames = self.progress["headers"][HeaderTypes.Measures]
        if not lnames and not vnames and not mnames:
            pass
        elif not lnames and not vnames:
            pass
        elif not vnames and not mnames:
            pass
        elif not lnames and not mnames:
            pass
        else:
            if not lnames:
                lnames = [None]
            if not vnames:
                vnames = [None]
            if not mnames:
                mnames = [None]
                # statistics
        snames = self.progress["headers"][HeaderTypes.Statistics]
        if not snames:
            snames = []
            # define get_cell
        row_type = self.progress["headers"]["rows"]
        col_type = self.progress["headers"]["columns"]
        if row_type == HeaderTypes.Locations:
            get_row = lambda (ln, vn, mn): ln
        elif row_type == HeaderTypes.Varieties:
            get_row = lambda (ln, vn, mn): vn
        elif row_type == HeaderTypes.Measures:
            get_row = lambda (ln, vn, mn): mn
        else:  # HeaderTypes.Data
            get_row = lambda (ln, vn, mn): None
        if col_type == HeaderTypes.Locations:
            get_col = lambda (ln, vn, mn): ln
        elif col_type == HeaderTypes.Varieties:
            get_col = lambda (ln, vn, mn): vn
        elif col_type == HeaderTypes.Measures:
            get_col = lambda (ln, vn, mn): mn
        else:  # HeaderTypes.Data
            get_col = lambda (ln, vn, mn): None

        def get_cell(ln, vn, mn):
            row = get_row((ln, vn, mn))
            col = get_col((ln, vn, mn))
            cell = (None, None)
            if row is None:
                tmp = col
                col = row
                row = tmp
            try:
                rows = self.progress["cells"][row]
            except KeyError:
                rows = []
            for col_val in rows:
                if col_val[0] == col:
                    cell = col_val
                    break
            return cell
            # collect entries

        for lname in lnames:
            location = self.progress["locations"][lname]
            location_name = "{}-{}".format(location["name"], location["planting_methods_text"])
            locations.append(location_name)
            summary_entries[lname] = {}
            for vname in vnames:
                variety = self.progress["varieties"][vname]
                variety_name = variety["name"]
                varieties.append(variety_name)
                for mname in mnames:
                    measure = self.progress["measures"][mname].copy()
                    measure_name = measure["fieldname"]
                    measures.append(measure_name)
                    val = get_cell(lname, vname, mname)[1]
                    if not isfloat(val):
                        val = None
                    measure["value"] = val
                    entry = {"location": location, "variety": variety, "measure": measure}
                    trial_entries[(location_name, variety_name, measure_name)] = entry
                    for sname in snames:
                        cells = self.progress["cells"][sname]
                        cell = (None, None)
                        for col_val in cells:
                            if col_val[0] == lname:
                                cell = col_val
                                header = lname
                                break
                            elif col_val[0] == vname:
                                cell = col_val
                                header = vname
                                break
                            elif col_val[0] == mname:
                                cell = col_val
                                header = mname
                                break
                        sentry = {"value": cell[1]}
                        summary_entries[header][sname] = sentry
                        # collect summary_entries
        for name in []:
            stats = {}
            stat = {}
            stat["value"] = sval
            stats[name] = stat
            summary_entries[location_name] = stats
            ## construct main table
        locations = sorted(list(set(locations)))
        varieties = sorted(list(set(varieties)))
        measures = sorted(list(set(measures)))
        table = []
        for v in varieties:
            row = []
            for l in locations:
                cell = []
                for m in measures:
                    try:
                        item = trial_entries[(l, v, m)]
                    except KeyError:
                        item = None
                    cell.append(item)
                row.append(cell)
            table.append(row)
            ## construct summary table (statistic entries)
        summary = []
        prettystatnames = []
        statistics = self.progress["statistics"]
        havestats = True
        if None in statistics and not statistics[None]:
            havestats = False
        if havestats:
            statmap = {}
            statnames = []
            # make a mapping from pretty names to user-input names
            for name in statistics:
                pretty = "{} ({})".format(statistics[name]["fieldname"], statistics[name]["alpha"])
                statmap[pretty] = name
                # sort pretty names
            prettystatnames = sorted(statmap.keys())
            # ensure we iterate using the ordering we just made over pretty names
            for name in prettystatnames:
                statnames.append(statmap[name])
            for name in statnames:
                row = []
                for l in lnames:  # only show stats that compare across locations
                    try:
                        val = summary_entries[l][name]["value"]
                    except KeyError:
                        val = None
                    row.append({"value": val, "comparing": statistics[name]["comparing"]})
                summary.append(row)
                ## bundle it up and send it along
        year = self.progress["date"]
        self.progress["trial_entries"] = trial_entries.values()
        # self.progress['table'] = table
        # self.progress['summary'] = summary
        self.save()
        return year, table, summary, prettystatnames, locations, varieties
Example #24
0
def swap_tracker_json(trmnt, user_id):

    my_buyin = Buy_ins.get_latest( user_id=user_id, tournament_id=trmnt.id )
    final_profit = 0

    swaps = Swaps.query.filter_by(
        sender_id = user_id,
        tournament_id = trmnt.id
    )

    # separate swaps by recipient id
    swaps_by_recipient = {}
    for swap in swaps:
        rec_id = str(swap.recipient_id)
        data = swaps_by_recipient.get( rec_id, [] )
        swaps_by_recipient[ rec_id ] = [ *data, swap ]
    
    # Loop through swaps to create swap tracker json and append to 'swaps_buyins'
    swaps_buyins = []
    for rec_id, swaps in swaps_by_recipient.items():
        recipient_buyin = Buy_ins.get_latest(
                user_id = rec_id,
                tournament_id = trmnt.id
            )

        # Catch ERRORS
        if recipient_buyin is None or my_buyin is None:
            swap_data_for_error_message = [{
                'recipient_name': f'{x.recipient_user.first_name} {x.recipient_user.last_name}',
                'sender_name': f'{x.sender_user.first_name} {x.sender_user.last_name}',
                'tournament_name': x.tournament.name 
            } for x in swaps]
            if recipient_buyin is None:
                return { 
                    'ERROR':'Recipient has swaps with user in this tournament but no buy-in',
                    'recipient buyin': None,
                    'swaps with user': swap_data_for_error_message }
            if my_buyin is None:
                return { 
                    'ERROR':'User has swaps in this tournament but no buy-in',
                    'buyin': None,
                    'user swaps': swap_data_for_error_message }

        # Structure and fill most properties for swap tracker json
        recipient_user = Profiles.query.get( rec_id )
        data = {
            'recipient_user': recipient_user.serialize(),
            'recipient_buyin': recipient_buyin.serialize(),
            'their_place': recipient_buyin.place,
            'you_won': my_buyin.winnings if my_buyin.winnings else 0,
            'they_won': recipient_buyin.winnings if recipient_buyin.winnings else 0,
            'available_percentage': recipient_user.available_percentage(trmnt.id),
            'agreed_swaps': [],
            'other_swaps': []
        }

        # Fill in properties: 'agreed_swaps' and 'other_swaps' lists
        you_owe_total = 0
        they_owe_total = 0
        for swap in swaps:
            single_swap_data = { **swap.serialize(),
                'counter_percentage': swap.counter_swap.percentage,
                'they_paid': swap.counter_swap.paid,
                'they_confirmed': swap.counter_swap.confirmed, }
            
            if swap.status._value_ == 'agreed':
                you_owe = ( float(my_buyin.winnings) * swap.percentage / 100 ) \
                    if isfloat(my_buyin.winnings) else 0
                they_owe = ( float(recipient_buyin.winnings) 
                    * swap.counter_swap.percentage / 100 ) \
                    if isfloat(recipient_buyin.winnings) else 0
                you_owe_total += you_owe
                they_owe_total += they_owe
                data['agreed_swaps'].append({
                    **single_swap_data,
                    'you_owe': you_owe,
                    'they_owe': they_owe
                })
            else:
                data['other_swaps'].append(single_swap_data)

        # Fill in final properties
        data['you_owe_total'] = you_owe_total
        data['they_owe_total'] = they_owe_total
        final_profit -= you_owe_total
        final_profit += they_owe_total

        # IF user doesn't owe anything to other guy, 
        # make all agreed swaps paid for themselves
        # if final_profit >= 0:
        #     for swap in data['agreed_swaps']:
        #         # swap['paid'] = True
        #         the_swap = Swaps.query.get( swap.id)
        #         print('the_swap', the_swap)
        #         the_swap.paid = True
        #         the_swap.paid_at = datetime.utcnow()
        #         the_swap.confirmed = True
        #         the_swap.confirmed_at = datetime.utcnow()


        # Append json
        swaps_buyins.append(data)

    return {
        'tournament': trmnt.serialize(),
        'my_buyin': my_buyin and my_buyin.serialize(),
        'buyins': swaps_buyins,
        'final_profit': final_profit
    }
Example #25
0
 def test_isfloat(self):
     true = utils.isfloat("123.45")
     false = utils.isfloat("hey")
     self.assertTrue(true)
     self.assertFalse(false)
Example #26
0
    def get_tournaments(user_id, id):

        if id == 'all':
            now = datetime.utcnow() - timedelta(days=1)

            # Filter past tournaments
            if request.args.get('history') == 'true':
                trmnts = Tournaments.get_history()

            # Filter current and future tournaments
            else:
                trmnts = Tournaments.get_live_upcoming()

            # Filter by name
            name = request.args.get('name')
            if name is not None:
                trmnts = trmnts.filter(Tournaments.name.ilike(f'%{name}%'))

            # Order by zip code
            zip = request.args.get('zip', '')
            if zip.isnumeric():
                path = os.environ['APP_PATH']
                with open(path + '/src/zip_codes.json') as zip_file:
                    data = json.load(zip_file)
                    zipcode = data.get(zip)
                    if zipcode is None:
                        raise APIException('Zipcode not in file', 500)
                    lat = zipcode['latitude']
                    lon = zipcode['longitude']

            # Order by user location
            else:
                lat = request.args.get('lat', '')
                lon = request.args.get('lon', '')

            if isfloat(lat) and isfloat(lon):
                trmnts = trmnts.order_by(
                    (db.func.abs(float(lon) - Tournaments.longitude) +
                     db.func.abs(float(lat) - Tournaments.latitude)).asc())

            # Order by ascending date
            elif request.args.get('asc') == 'true':
                trmnts = trmnts.order_by(Tournaments.start_at.asc())

            # Order by descending date
            elif request.args.get('desc') == 'true':
                trmnts = trmnts.order_by(Tournaments.start_at.desc())

            # Pagination
            offset, limit = utils.resolve_pagination(request.args)
            trmnts = trmnts.offset(offset).limit(limit)

            return jsonify([
                actions.swap_tracker_json(trmnt, user_id) for trmnt in trmnts
            ]), 200

        # Single tournament by id
        elif id.isnumeric():
            trmnt = Tournaments.query.get(int(id))
            if trmnt is None:
                raise APIException('Tournament not found', 404)

            return jsonify(actions.swap_tracker_json(trmnt, user_id)), 200

        raise APIException('Invalid id', 400)
Example #27
0
    def on_main_button_click(self, sender, sender_name):
        if sender_name != 'OK' or any(entry is None for entry in self.res.itervalues()) or self.subtitleModel.is_empty() or not(isfloat(self.res['originalMS']) or isfloat(self.res['targetMS'])):
            self.destroy()
            return
        
        # Create list of subtitles to apply changes
        applyItems = []
        if self.res['applyTo'] == 'all lines':
            for item in self.subtitleModel.get_model():
                applyItems.append(item[0])
        else:
            self.get_tv_selection()
            if len(self.tvSelectionList) == 0:
                self.destroy()
                return
            applyItems = self.tvSelectionList[:]
        
        origMS = int(self.res['originalMS'])
        targetMS = int(self.res['targetMS'])

        for item in applyItems:
            if (self.res['condition'] == '<' and not(item.duration < origMS)) or (self.res['condition'] == '<=' and not(item.duration < origMS)) or (self.res['condition'] == '>' and not(item.duration > origMS)) or (self.res['condition'] == '>=' and not(item.duration >= origMS)) or (self.res['condition'] == '=' and not(item.duration == origMS)):
                continue

            # Check if new duration is less than the minimum allowed
            minDuration = int ( (20.0 * (item.calc_target_duration() - 500) / 26.9999999999) + 500 ) + 1
            targetDuration = item.duration + targetMS * (1 if self.res['operation'] == 'add' else -1)
            targetDuration = targetDuration if targetDuration >= minDuration else minDuration
            targetDuration = targetDuration if targetDuration >= 1000 else 1000

            # Check if gap is not violated
            newStop = int(item.stopTime) + (int(targetDuration) - int(item.duration))
            nextSub = self.subtitleModel.get_next(item)
            if nextSub is not None and newStop > nextSub.startTime - 120:
                newStop = nextSub.startTime - 120

            # Check if change in stopTime violates min duration
            if int(newStop) - int(item.startTime) < int(minDuration):
                continue

            # Finally Apply newStop
            self.changeList.append( (item, int(item.startTime), int(item.stopTime), int(item.startTime), int(newStop)) )
            item.stopTime = int(newStop)

        self.destroy()
    def get_tournaments(user_id, id):

        # List Flights
        if id == 'all':

            # Order by date: ascending or descending
            order_method = None
            if request.args.get('asc') == 'true':
                order_method = Flights.start_at.asc()
            elif request.args.get('desc') == 'true':
                order_method = Flights.start_at.desc()

            # Filter past flights and order by default asc
            if request.args.get('history') == 'true':
                flights = Flights.get(history=True)
                flights = flights.order_by(Flights.start_at.desc(
                ) if order_method is None else order_method)

            # Filter current and future flights and order by default desc
            else:
                flights = Flights.get(history=False)
                flights = flights.order_by(Flights.start_at.asc(
                ) if order_method is None else order_method)

            # Filter by name
            name = request.args.get('name')
            if name is not None:
                flights = flights.filter(
                    Flights.tournament.has(
                        Tournaments.name.ilike(f'%{name}%')))

            # Get zip code LAT LON
            zip = request.args.get('zip', '')
            if zip.isnumeric():
                path = os.environ['APP_PATH']
                with open(path + '/src/zip_codes.json') as zip_file:
                    data = json.load(zip_file)
                    zipcode = data.get(zip)
                    if zipcode is None:
                        raise APIException('Zipcode not in file', 500)
                    lat = zipcode['latitude']
                    lon = zipcode['longitude']

            # Get user LAT LON
            else:
                lat = request.args.get('lat', '')
                lon = request.args.get('lon', '')

            # Order flights by distance, whithin the day
            if isfloat(lat) and isfloat(lon):
                flights = [{
                    'flight':
                    f,
                    'distance':
                    utils.distance(origin=[float(lat), float(lon)],
                                   destination=[
                                       f.tournament.latitude,
                                       f.tournament.longitude
                                   ])
                } for f in flights]

                flights = sorted(flights,
                                 key=cmp_to_key(utils.sort_by_location))

                # Pagination
                offset, limit = utils.resolve_pagination(request.args)
                flights = flights[offset:offset + limit]

                return jsonify([{
                    **x['flight'].serialize(), 'casino':
                    x['flight'].tournament.casino,
                    'address':
                    x['flight'].tournament.address,
                    'city':
                    x['flight'].tournament.city,
                    'state':
                    x['flight'].tournament.state,
                    'zip_code':
                    x['flight'].tournament.zip_code,
                    'buy_in':
                    Buy_ins.get_latest(user_id, x['flight'].tournament_id)
                    is not None,
                    'distance':
                    x['distance']
                } for x in flights]), 200

            else:
                # Pagination
                offset, limit = utils.resolve_pagination(request.args)
                flights = flights.offset(offset).limit(limit)

                return jsonify([{
                    **f.serialize(), 'casino':
                    f.tournament.casino,
                    'address':
                    f.tournament.address,
                    'city':
                    f.tournament.city,
                    'state':
                    f.tournament.state,
                    'zip_code':
                    f.tournament.zip_code,
                    'buy_in':
                    Buy_ins.get_latest(user_id, f.tournament_id) is not None
                } for f in flights]), 200

        # Single tournament by id
        elif id.isnumeric():
            trmnt = Tournaments.query.get(int(id))
            if trmnt is None:
                raise APIException('Tournament not found', 404)

            return jsonify(actions.swap_tracker_json(trmnt, user_id)), 200

        raise APIException('Invalid id', 400)
Example #29
0
    def on_main_button_click(self, sender, sender_name):
        if sender_name != 'OK':
            self.destroy()
            return

        if self.res['applyToSubs'] is None or self.res[
                'milliseconds1'] is None or self.res[
                    'operation1'] is None or self.res['applyToTime1'] is None:
            self.destroy()
            return

        if self.res['applyToTime1'] == 'Start Time' and (
                self.res['operation2'] is not None
                and not isfloat(self.res['milliseconds2'])):
            self.destroy()
            return

        if not isfloat(self.res['milliseconds1']):
            self.destroy()
            return

        # Create list of subtitles to apply changes
        applyItems = []
        if self.res['applyToSubs'] == 'all lines':
            for item in self.subtitleModel.get_model():
                applyItems.append(item[0])
        else:
            self.get_tv_selection()
            if len(self.tvSelectionList) == 0:
                self.destroy()
                return
            applyItems = self.tvSelectionList[:]

        if self.res['milliseconds1'] is not None:
            milliseconds1 = int(self.res['milliseconds1']) * (
                1 if self.res['operation1'] == 'add' else -1)
        else:
            milliseconds1 = 0
        if self.res['milliseconds2'] is not None:
            milliseconds2 = int(self.res['milliseconds2']) * (
                1 if self.res['operation1'] == 'add' else -1)
        else:
            milliseconds2 = 0

        operation1Items = applyItems[:]
        if milliseconds1 > 0:
            operation1Items.reverse()

        for item in applyItems:
            newBound = [None, None]
            if self.res['applyToTime1'] == 'Start Time':
                newBound[0] = int(item.startTime) + milliseconds1
                newBound[1] = int(item.stopTime) + milliseconds2
            elif self.res['applyToTime1'] == 'Stop Time':
                newBound[0] = int(item.startTime)
                newBound[1] = int(item.stopTime) + milliseconds1
            elif self.res['applyToTime1'] == 'Both Start and Stop Times':
                newBound[0] = int(item.startTime) + milliseconds1
                newBound[1] = int(item.stopTime) + milliseconds1
            else:
                continue

            nextSub = self.subtitleModel.get_next(item)
            prevSub = self.subtitleModel.get_prev(item)

            if newBound[0] < 0:
                return
            if nextSub is not None and newBound[1] > nextSub.startTime - 120:
                continue
            if prevSub is not None and newBound[0] < prevSub.stopTime + 120:
                continue
            if newBound[0] > newBound[1] or newBound[1] - newBound[0] < 1000:
                continue
            self.changeList.append(
                (item, int(item.startTime), int(item.stopTime),
                 int(newBound[0]), int(newBound[1])))
            item.startTime = int(newBound[0])
            item.stopTime = int(newBound[1])
        self.destroy()
Example #30
0
def plot_feature_contributions_surgery_class(X,
                                             y,
                                             feature_index,
                                             fcs,
                                             attributes,
                                             class_of_interest,
                                             title=None):
    surgery_index = np.where(attributes == 'Q44071_snCplexoAt')[0][0]

    if (not utils.isint(X[utils.firstNotNan(
            X[:, feature_index])][feature_index]) and not utils.isfloat(
                X[utils.firstNotNan(X[:, feature_index])][feature_index])):
        values = [i for i in set(X[:, feature_index]) if not utils.isnan(i)
                  ] + [np.nan]

        x_surgery = []
        surgery_colors = []
        x_no_surgery = []
        no_surgery_colors = []
        x_nan = []
        nan_colors = []
        y_surgery = []
        y_no_surgery = []
        y_nan = []

        contributions = {}

        for i in range(X.shape[0]):

            if (feature_index in fcs[i].keys()):

                if (X[i][surgery_index] == 'S' or X[i][surgery_index] == 'Y'):
                    x_surgery.append(fcs[i][feature_index][class_of_interest])
                    y_surgery.append(values.index(X[i][feature_index]))
                    if (y[i] == class_of_interest):
                        surgery_colors.append('blue')
                    else:
                        surgery_colors.append('red')

                elif (utils.isnan(X[i][surgery_index])):
                    x_nan.append(fcs[i][feature_index][class_of_interest])
                    #this is necessary because of weird behavior when X[i][feature_index] is nan
                    #and for some reason it says that nan is not values
                    y_nan.append(len(values) - 1)
                    if (y[i] == class_of_interest):
                        nan_colors.append('blue')
                    else:
                        nan_colors.append('red')
                else:
                    x_no_surgery.append(
                        fcs[i][feature_index][class_of_interest])
                    y_no_surgery.append(values.index(X[i][feature_index]))
                    if (y[i] == class_of_interest):
                        no_surgery_colors.append('blue')
                    else:
                        no_surgery_colors.append('red')

                # if(X[i][feature_index] not in contributions.keys()):
                #     contributions[X[i][feature_index]] = [fcs[i][feature_index][class_of_interest]]
                # else:
                #     contributions[X[i][feature_index]].append(fcs[i][feature_index][class_of_interest])
        coi = str(class_of_interest)
        ax = plt.subplot(111)
        ax.scatter(x_surgery,
                   y_surgery,
                   marker='o',
                   s=60,
                   edgecolors=surgery_colors,
                   facecolors='none')
        ax.scatter(x_no_surgery,
                   y_no_surgery,
                   marker='x',
                   s=60,
                   edgecolors=no_surgery_colors,
                   facecolors='none')
        ax.scatter(x_nan,
                   y_nan,
                   marker='d',
                   s=60,
                   edgecolors=nan_colors,
                   facecolors='none')
        plt.xlabel('feature contribution')
        plt.ylabel('values of feature %r' % attributes[feature_index])
        ax.set_yticks(np.array(range(len(values) + 2)) - 1)
        ax.set_yticklabels([str('')] + values + [str('')])
        red_patch = mpatches.Patch(color='red')
        blue_patch = mpatches.Patch(color='blue')
        xmarker = mlines.Line2D([], [],
                                color='black',
                                marker='x',
                                markersize=10,
                                linestyle='None')
        omarker = mlines.Line2D([], [],
                                color='black',
                                marker='o',
                                markersize=10,
                                linestyle='None',
                                markerfacecolor='None',
                                markeredgecolor='black')
        #plt.legend(handles=[red_patch,blue_patch])

        plt.legend([red_patch, blue_patch, xmarker, omarker], [
            'Classe da instância ≠ ' + coi, 'Classe da instância = ' + coi,
            'Não passou por cirurgia', 'Passou por cirurgia'
        ],
                   numpoints=1,
                   fontsize='small')
        plt.show()

    else:

        values = sorted([
            round(i, 4) for i in (set(X[:, feature_index]))
            if not utils.isnan(i)
        ])  # + [np.nan]
        print(values)
        nan_index = values[-1] - values[-2]
        x_surgery = []
        surgery_colors = []
        x_no_surgery = []
        no_surgery_colors = []
        x_nan = []
        nan_colors = []
        y_surgery = []
        y_no_surgery = []
        y_nan = []

        for i in range(X.shape[0]):
            if (feature_index in fcs[i].keys()):
                if (X[i][surgery_index] == 'S' or X[i][surgery_index] == 'Y'):
                    x_surgery.append(fcs[i][feature_index][class_of_interest])
                    y_surgery.append((X[i][feature_index]))
                    if (y[i] == class_of_interest):
                        surgery_colors.append('blue')
                    else:
                        surgery_colors.append('red')
                elif (utils.isnan(X[i][surgery_index])):
                    x_nan.append(fcs[i][feature_index][class_of_interest])
                    #this is necessary because of weird behavior when X[i][feature_index] is nan
                    #and for some reason it says that nan is not values
                    y_nan.append(values[-1] + nan_index)
                    if (y[i] == class_of_interest):
                        nan_colors.append('blue')
                    else:
                        nan_colors.append('red')
                else:
                    x_no_surgery.append(
                        fcs[i][feature_index][class_of_interest])
                    y_no_surgery.append((X[i][feature_index]))
                    if (y[i] == class_of_interest):
                        no_surgery_colors.append('blue')
                    else:
                        no_surgery_colors.append('red')
        coi = str(class_of_interest)
        fig, ax = plt.subplots()
        ax.scatter(x_surgery,
                   y_surgery,
                   marker='o',
                   s=60,
                   facecolors='none',
                   edgecolors=surgery_colors)
        ax.scatter(x_no_surgery,
                   y_no_surgery,
                   marker='x',
                   s=60,
                   edgecolors=no_surgery_colors)
        ax.scatter(x_nan,
                   y_nan,
                   marker='d',
                   s=60,
                   facecolors='none',
                   edgecolors=nan_colors)
        fig.canvas.draw()
        labels = [''] + [item.get_text()
                         for item in ax.get_yticklabels()] + ['']
        if (values[-1] + nan_index < ax.get_yticks()[-1]):
            plt.yticks(
                [values[0] - nan_index] +
                sorted(list(ax.get_yticks()) + [values[-1] + nan_index]))
        else:
            plt.yticks([values[0] - nan_index] + sorted(
                list(ax.get_yticks()) +
                [values[-1] + nan_index, values[-1] + 2 * nan_index]))
        labels[-2] = 'nan'

        plt.xlabel('feature contribution')
        plt.ylabel('values of feature %r' % attributes[feature_index])
        ax.set_yticklabels(labels)
        red_patch = mpatches.Patch(color='red')
        blue_patch = mpatches.Patch(color='blue')
        xmarker = mlines.Line2D([], [],
                                color='black',
                                marker='x',
                                markersize=10,
                                label='Bla',
                                linestyle='None')
        omarker = mlines.Line2D([], [],
                                color='black',
                                marker='o',
                                markersize=10,
                                label='Bla',
                                linestyle='None',
                                markerfacecolor='None',
                                markeredgecolor='black')
        #plt.legend(handles=[red_patch,blue_patch])
        plt.legend([red_patch, blue_patch, xmarker, omarker], [
            'Classe da instância ≠ ' + coi, 'Classe da instância = ' + coi,
            'Não passou por cirurgia', 'Passou por cirurgia'
        ],
                   numpoints=1,
                   fontsize='small')
        plt.show()

    if (title is not None):
        plt.savefig(title)
        plt.close()
        f = open(title, 'w')
        f.write('X=' + str(X))
        f.write('\ny=' + str(y))
        f.write('\nfcs=' + str(fcs))
        f.write('\nfeatures=' + str(attributes))
        f.write('\nfeature_index=' + str(feature_index))
        f.write('\nvalues=' + str(values))
        f.write('\nx_surgery=' + str(x_surgery))
        f.write('\ny_surgery=' + str(y_surgery))
        f.write('\nsurgery_colors=' + str(surgery_colors))
        f.write('\nx_no_surgery=' + str(x_no_surgery))
        f.write('\ny_no_surgery=' + str(y_no_surgery))
        f.write('\nno_surgery_colors=' + str(no_surgery_colors))
        f.write('\nx_nan=' + str(x_nan))
        f.write('\ny_nan=' + str(y_nan))
        f.write('\nnan_colors=' + str(nan_colors))
Example #31
0
def plot_feature_contributions(X,
                               feature_index,
                               fcs,
                               attributes,
                               class_of_interest,
                               title=None):

    if (not utils.isint(X[utils.firstNotNan(
            X[:, feature_index])][feature_index]) and not utils.isfloat(
                X[utils.firstNotNan(X[:, feature_index])][feature_index])):
        values = [i for i in set(X[:, feature_index]) if not utils.isnan(i)
                  ] + [np.nan]

        pos_fcs = []
        neg_fcs = []
        pos_values = []
        neg_values = []
        zero_fcs = []
        zero_values = []
        contributions = {}

        for i in range(X.shape[0]):

            if (feature_index in fcs[i].keys()):
                if (fcs[i][feature_index][class_of_interest] > 0):
                    pos_fcs.append(fcs[i][feature_index][class_of_interest])
                    #this is necessary because of weird behavior when X[i][feature_index] is nan
                    #and for some reason it says that nan is not values
                    if (utils.isnan(X[i][feature_index])):
                        pos_values.append(len(values) - 1)
                    else:
                        pos_values.append(values.index(X[i][feature_index]))
                elif (fcs[i][feature_index][class_of_interest] == 0):
                    zero_fcs.append(0)
                    if (utils.isnan(X[i][feature_index])):
                        zero_values.append(len(values) - 1)
                    else:
                        zero_values.append(values.index(X[i][feature_index]))
                else:
                    neg_fcs.append(fcs[i][feature_index][class_of_interest])
                    if (utils.isnan(X[i][feature_index])):
                        neg_values.append(len(values) - 1)
                    else:
                        neg_values.append(values.index(X[i][feature_index]))
                if (X[i][feature_index] not in contributions.keys()):
                    contributions[X[i][feature_index]] = [
                        fcs[i][feature_index][class_of_interest]
                    ]
                else:
                    contributions[X[i][feature_index]].append(
                        fcs[i][feature_index][class_of_interest])

        print('Contributions:')
        for value in contributions.keys():
            print('Value %r' % value)
            print(
                '\nMean: %r Variance: %r' %
                (np.mean(contributions[value]), np.var(contributions[value])))

        c = (contributions.items())
        boxplot([a[1] for a in c], [a[0] for a in c], title=None)

        ax = plt.subplot(111)
        plt.plot(pos_fcs, pos_values, 'x', color='blue')
        plt.plot(neg_fcs, neg_values, 'x', color='red')
        plt.plot(zero_fcs, zero_values, 'x', color='black')
        plt.xlabel('feature contribution')
        plt.ylabel('values of feature %r' % attributes[feature_index])
        ax.set_yticks(np.array(range(len(values) + 2)) - 1)
        ax.set_yticklabels([str('')] + values + [str('')])
        plt.show()

    else:

        values = sorted([
            round(i, 4) for i in (set(X[:, feature_index]))
            if not utils.isnan(i)
        ])  # + [np.nan]

        nan_index = values[-1] - values[-2]
        pos_fcs = []
        neg_fcs = []
        pos_values = []
        neg_values = []
        zero_fcs = []
        zero_values = []
        contributions = {}

        for i in range(X.shape[0]):
            if (feature_index in fcs[i].keys()):

                if (fcs[i][feature_index][class_of_interest] > 0):
                    pos_fcs.append(fcs[i][feature_index][class_of_interest])
                    #this is necessary because of weird behavior when X[i][feature_index] is nan
                    #and for some reason it says that nan is not values
                    if (utils.isnan(X[i][feature_index])):
                        pos_values.append(values[-1] + nan_index)
                    else:
                        pos_values.append(X[i][feature_index])
                elif (fcs[i][feature_index][class_of_interest] == 0):
                    zero_fcs.append(0)
                    if (utils.isnan(X[i][feature_index])):
                        zero_values.append(values[-1] + nan_index)
                    else:
                        zero_values.append(X[i][feature_index])
                else:
                    neg_fcs.append(fcs[i][feature_index][class_of_interest])
                    if (utils.isnan(X[i][feature_index])):
                        neg_values.append(values[-1] + nan_index)
                    else:
                        neg_values.append((X[i][feature_index]))
                if (utils.isnan(X[i][feature_index])):
                    if ('nan' in contributions.keys()):
                        contributions['nan'].append(
                            fcs[i][feature_index][class_of_interest])
                    else:
                        contributions['nan'] = [
                            fcs[i][feature_index][class_of_interest]
                        ]
                elif (X[i][feature_index] in contributions.keys()):
                    contributions[(X[i][feature_index])].append(
                        fcs[i][feature_index][class_of_interest])
                else:
                    contributions[(X[i][feature_index])] = [
                        fcs[i][feature_index][class_of_interest]
                    ]

        print('Contributions:')
        for value in contributions.keys():
            print('Value %r' % value)
            print(
                'Mean: %r Variance: %r' %
                (np.mean(contributions[value]), np.std(contributions[value])))
        c = (contributions.items())
        boxplot([a[1] for a in c], [a[0] for a in c], title=None)
        fig, ax = plt.subplots()
        plt.plot(pos_fcs, pos_values, 'x', color='blue')
        plt.plot(neg_fcs, neg_values, 'x', color='red')
        plt.plot(zero_fcs, zero_values, 'x', color='black')
        fig.canvas.draw()
        labels = [''] + [item.get_text()
                         for item in ax.get_yticklabels()] + ['']
        if (values[-1] + nan_index < ax.get_yticks()[-1]):
            plt.yticks(
                [values[0] - nan_index] +
                sorted(list(ax.get_yticks()) + [values[-1] + nan_index]))
        else:
            plt.yticks([values[0] - nan_index] + sorted(
                list(ax.get_yticks()) +
                [values[-1] + nan_index, values[-1] + 2 * nan_index]))
        labels[-2] = 'nan'

        plt.xlabel('feature contribution')
        plt.ylabel('values of feature %r' % attributes[feature_index])
        ax.set_yticklabels(labels)

        plt.show()

    if (title is not None):
        plt.savefig(title)
        plt.close()
Example #32
0
dummy = False
transform = False
use_text = False

print('Testing utils...')
assert (utils.isint(10))
assert (utils.isint('50'))
assert (utils.isint('-999'))
assert (not utils.isint(1.0))
assert (not utils.isint('50.0'))
assert (utils.isint(True))
assert (not utils.isint('aba'))
assert (not utils.isint('a?a'))
assert (not utils.isint('49.x'))
assert (not utils.isfloat('0.x'))
assert (utils.isfloat('0.0'))
assert (utils.isfloat('12.984'))
assert (utils.isfloat('-0.4'))
assert (not utils.isfloat('9'))

original_attributes = np.array(
    ['Outlook', 'Temp', 'Humidity', 'Windy?', 'Class'])
data = pd.DataFrame(
    [['SUNNY', 75, 70, 'T', 'PLAY'], ['SUNNY', 80, 90, 'T', "DON'T PLAY"],
     ['SUNNY', 85, 85, 'F', "DON'T PLAY"], [
         'SUNNY', 72, 95, 'F', "DON'T PLAY"
     ], ['SUNNY', 69, 70, 'F', 'PLAY'], ['OVERCAST', 72, 90, 'T', 'PLAY'],
     ['OVERCAST', 83, 78, 'F', 'PLAY'], ['OVERCAST', 64, 65, 'T', 'PLAY'],
     ['OVERCAST', 81, 75, 'F', 'PLAY'], ['RAIN', 71, 80, 'T', "DON'T PLAY"],
     ['RAIN', 65, 70, 'T', "DON'T PLAY"], ['RAIN', 75, 80, 'F', 'PLAY'],
Example #33
0
 def write_database(self):
     # create SignificanceEntry
     significance_entries = []
     name_sigentry = {}
     statistics = self.progress['statistics']
     # TODO: BUG: if a location has no entries, but has stat entries,
     # TODO: it will not show up in the preview but will be written here
     # TODO: (kinda harmless though, since it won't have any trials mapped to it)
     for name in statistics:
         if not statistics[name]:
             continue
         fieldname = statistics[name]['fieldname']
         alpha = statistics[name]['alpha']
         if not alpha:
             alpha = None
         comparing = statistics[name]['comparing']
         try:
             cells = self.progress['cells'][name]
         except KeyError:
             cells = []
         for header, value in cells:
             if not value or not isfloat(value):
                 continue
             sigentry = models.SignificanceEntry(
                 comparing=comparing,
                 method=fieldname,
                 alpha=alpha,
                 value=value,
             )
             significance_entries.append(sigentry)
             try:
                 name_sigentry[header].append(sigentry)
             except KeyError:
                 name_sigentry[header] = [sigentry]
     for sigentry in significance_entries:
         sigentry.save()
     # create Date
     year = int(self.progress['date'])
     date_pd = datetime.date(year=year, month=5, day=1)
     date_hd = datetime.date(year=year, month=8, day=1)
     try:
         plant_date = models.Date.objects.get(date=date_pd)
     except:
         plant_date = models.Date(date=date_pd)
         plant_date.save()
     try:
         harvest_date = models.Date.objects.get(date=date_hd)
     except:
         harvest_date = models.Date(date=date_hd)
         harvest_date.save()
     # create/update TrialEntry objects
     entries = []
     lpvh_entries = {}
     location_names = []
     variety_names = []
     planting_methods = []
     for trial in models.TrialEntry.objects.filter(
             harvest_date=harvest_date):
         lpvh = (trial.location.pk, trial.planting_method_tags.pk,
                 trial.variety.pk, trial.harvest_date.pk)
         lpvh_entries[lpvh] = trial
     for entry in self.progress['trial_entries']:
         location = entry['location']['pk']
         planting_methods = entry['location']['planting_methods']
         variety = entry['variety']['pk']
         lpvh = (location, planting_methods, variety, harvest_date.pk)
         try:
             trial = lpvh_entries[lpvh]
         except KeyError:
             trial = models.TrialEntry(
                 location_id=location,
                 variety_id=variety,
                 planting_method_tags_id=planting_methods,
                 harvest_date=harvest_date,
                 plant_date=plant_date,
             )
             lpvh_entries[lpvh] = trial
         measure = entry['measure']['fieldname']
         value = entry['measure']['value']
         if value is None:
             continue
         setattr(trial, measure, value)
         entries.append(trial)
         trial.save()
         # grab info for a later requery of these trials
         location_names.append(trial.location.name)
         variety_names.append(trial.variety.name)
         planting_methods.append(trial.planting_method_tags.planting_method)
     # attach TrialEntry objects to SignificanceEntry objects
     for name, sigentries in name_sigentry.items():
         # name is a user-entered string, lookup in our mappings
         # TODO: assume name is a location for now
         location = self.progress['locations'][name]['pk']
         planting_methods = self.progress['locations'][name][
             'planting_methods']
         # TODO: avoid the following database call
         trials = list(
             models.TrialEntry.objects.filter(location_id=location).filter(
                 planting_method_tags_id=planting_methods).filter(
                     harvest_date=harvest_date))
         for sigentry in sigentries:
             sigentry.trials.add(*trials)
             sigentry.save()
     # save TrialEntry query for later editing
     self.progress['submitted']['locations'] = list(set(location_names))
     self.progress['submitted']['varieties'] = list(set(variety_names))
     self.progress['submitted']['planting_methods'] = list(
         set(planting_methods))
     self.progress['submitted']['harvest_date'] = {
         'year': harvest_date.date.year,
         'month': harvest_date.date.month,
         'day': harvest_date.date.day,
     }
     self.save()
     self.model.submitted = True
     self.model.save()
Example #34
0
 def on_zoomEntry_change(self, sender):
     if isfloat(sender.get_text()):
         if float(sender.get_text()) == 0:
             return
         self.preferences['Zoom'] = float(sender.get_text())
Example #35
0
 def _fmt_pair(k,v):
     if isfloat(v):
         v = sfrounds(v,nsigfigs)
     else:
         v = repr(v)
     return '{0}={1}'.format(k,v)
Example #36
0
    def prepare_table(self):
        ## collect entered data
        trial_entries = {}
        summary_entries = {}
        locations = []
        varieties = []
        measures = []
        # locations, varieties, measures
        lnames = self.progress['headers'][HeaderTypes.Locations]
        vnames = self.progress['headers'][HeaderTypes.Varieties]
        mnames = self.progress['headers'][HeaderTypes.Measures]
        if not lnames and not vnames and not mnames:
            pass
        elif not lnames and not vnames:
            pass
        elif not vnames and not mnames:
            pass
        elif not lnames and not mnames:
            pass
        else:
            if not lnames:
                lnames = [None]
            if not vnames:
                vnames = [None]
            if not mnames:
                mnames = [None]
        # statistics
        snames = self.progress['headers'][HeaderTypes.Statistics]
        if not snames:
            snames = []
        # define get_cell
        row_type = self.progress['headers']['rows']
        col_type = self.progress['headers']['columns']
        if row_type == HeaderTypes.Locations:
            get_row = lambda (ln, vn, mn): ln
        elif row_type == HeaderTypes.Varieties:
            get_row = lambda (ln, vn, mn): vn
        elif row_type == HeaderTypes.Measures:
            get_row = lambda (ln, vn, mn): mn
        else:  # HeaderTypes.Data
            get_row = lambda (ln, vn, mn): None
        if col_type == HeaderTypes.Locations:
            get_col = lambda (ln, vn, mn): ln
        elif col_type == HeaderTypes.Varieties:
            get_col = lambda (ln, vn, mn): vn
        elif col_type == HeaderTypes.Measures:
            get_col = lambda (ln, vn, mn): mn
        else:  # HeaderTypes.Data
            get_col = lambda (ln, vn, mn): None

        def get_cell(ln, vn, mn):
            row = get_row((ln, vn, mn))
            col = get_col((ln, vn, mn))
            cell = (None, None)
            if row is None:
                tmp = col
                col = row
                row = tmp
            try:
                rows = self.progress['cells'][row]
            except KeyError:
                rows = []
            for col_val in rows:
                if col_val[0] == col:
                    cell = col_val
                    break
            return cell

        # collect entries
        for lname in lnames:
            location = self.progress['locations'][lname]
            location_name = '{}-{}'.format(location['name'],
                                           location['planting_methods_text'])
            locations.append(location_name)
            summary_entries[lname] = {}
            for vname in vnames:
                variety = self.progress['varieties'][vname]
                variety_name = variety['name']
                varieties.append(variety_name)
                for mname in mnames:
                    measure = self.progress['measures'][mname].copy()
                    measure_name = measure['fieldname']
                    measures.append(measure_name)
                    val = get_cell(lname, vname, mname)[1]
                    if not isfloat(val):
                        val = None
                    measure['value'] = val
                    entry = {
                        'location': location,
                        'variety': variety,
                        'measure': measure,
                    }
                    trial_entries[(location_name, variety_name,
                                   measure_name)] = entry
                    for sname in snames:
                        cells = self.progress['cells'][sname]
                        cell = (None, None)
                        for col_val in cells:
                            if col_val[0] == lname:
                                cell = col_val
                                header = lname
                                break
                            elif col_val[0] == vname:
                                cell = col_val
                                header = vname
                                break
                            elif col_val[0] == mname:
                                cell = col_val
                                header = mname
                                break
                        sentry = {
                            'value': cell[1],
                        }
                        summary_entries[header][sname] = sentry
        # collect summary_entries
        for name in []:
            stats = {}
            stat = {}
            stat['value'] = sval
            stats[name] = stat
            summary_entries[location_name] = stats
        ## construct main table
        locations = sorted(list(set(locations)))
        varieties = sorted(list(set(varieties)))
        measures = sorted(list(set(measures)))
        table = []
        for v in varieties:
            row = []
            for l in locations:
                cell = []
                for m in measures:
                    try:
                        item = trial_entries[(l, v, m)]
                    except KeyError:
                        item = None
                    cell.append(item)
                row.append(cell)
            table.append(row)
        ## construct summary table (statistic entries)
        summary = []
        prettystatnames = []
        statistics = self.progress['statistics']
        havestats = True
        if None in statistics and not statistics[None]:
            havestats = False
        if havestats:
            statmap = {}
            statnames = []
            # make a mapping from pretty names to user-input names
            for name in statistics:
                pretty = '{} ({})'.format(statistics[name]['fieldname'],
                                          statistics[name]['alpha'])
                statmap[pretty] = name
            # sort pretty names
            prettystatnames = sorted(statmap.keys())
            # ensure we iterate using the ordering we just made over pretty names
            for name in prettystatnames:
                statnames.append(statmap[name])
            for name in statnames:
                row = []
                for l in lnames:  # only show stats that compare across locations
                    try:
                        val = summary_entries[l][name]['value']
                    except KeyError:
                        val = None
                    row.append({
                        'value': val,
                        'comparing': statistics[name]['comparing']
                    })
                summary.append(row)
        ## bundle it up and send it along
        year = self.progress['date']
        self.progress['trial_entries'] = trial_entries.values()
        #self.progress['table'] = table
        #self.progress['summary'] = summary
        self.save()
        return year, table, summary, prettystatnames, locations, varieties