def to_R(self, ds, dates_as_factor, suppress_all_value=True, **kwargs): if not self.colname: return col = ds[self.colname] if col.is_datetimetype(): rdata = rconv.Missing_Date_to_R(col) if not dates_as_factor: return rdata elif MA.isMaskedArray(col.data): if self.missing_as_none: rdata = MA.filled(col.data, sys.maxint) else: if Numeric.alltrue(col.data.mask()): raise PlotError('%r: all data-points masked' % col.name) rdata = rconv.MA_to_R(col.data) else: rdata = col.data if col.is_discrete(): if 0 and suppress_all_value: row_vecs = [ vec for value, vec in col.inverted.items() if value != col.all_value ] row_vec = union(*row_vecs) rdata = MA.take(rdata, row_vec) return self.discrete_col_to_R(col, rdata, suppress_all_value) else: return self.continuous_col_to_R(col, rdata)
def from_summset(cls, ds, shaped_like=None): self = cls(ds.name) st = time.time() cols = ds.get_columns() if shaped_like is not None: for axis in xtab_axes(shaped_like): try: col = ds[axis.name] except KeyError: pass else: self.axes.append(CrossTabAxis.from_col(col, axis.values)) cols.remove(col) for col in cols: if col.is_discrete() and not col.name.startswith('_'): self.axes.append(CrossTabAxis.from_col(col)) if not self.axes: raise Error('dataset %r must have at least one discrete column' % (ds.name,)) indices = [axis.indices.filled() for axis in self.axes] masks = [axis.indices.mask() for axis in self.axes] map = MA.transpose(MA.array(indices, mask=masks)) shape = self.get_shape() for col in ds.get_columns(): if col.is_scalar(): self.add_table(col.name, data=self.from_vector(map, col.data, shape), label=col.label) elapsed = time.time() - st soom.info('%r crosstab generation took %.3f, %.1f rows/s' % (self.name, elapsed, len(map) / elapsed)) return self
def testMultipleDimensionCreation (self): 'Test creation of multidimensional arrays.' alist = [[2,3],[4,5]] x = MA.array(alist) assert x.shape == (2,2) y = MA.array([x, x + 1, x + 2]) assert y.shape == (3,2,2)
def accumulate24Hourly(data): """Returns 12-hourly data accumulated to 24-hours.""" newTimeValues=[] taxis=data.getTime() tunits=data.units print len(data.getTime()) newarray=[] for i in range((tlen/2)): p1=data(time=slice(i,i+1)) p2=data(time=slice(i+1,i+2)) accum=p1+p2 newarray.append(accum) newTimeValues.append(p2.getTime()[0]) array=MA.concatenate(newarray) array=MA.array(array, 'f', fill_value=data.getMissing()) axes=data.getAxisList() newTimeAxis=cdms.createAxis(newTimeValues) newTimeAxis.units=tunits newTimeAxis.designateTime() newTimeAxis.id=newTimeAxis.long_name=newTimeAxis.title="time" newaxes=[newTimeAxis]+axes[1:] var=cdms.createVariable(array, axes=newaxes, id=data.id) for att in ("units", "long_name"): setattr(var, att, getattr(data, att)) return var
def testConcatenate (self): "test concatenate" assert eq (MA.concatenate((self.a[:3], self.a[3:])), [0,1,2,3,4,5]) m = self.m assert eq (MA.concatenate((m, m)), [[1,2,3], [11,12,13], [1,2,3], [11,12,13]]) assert eq (MA.concatenate((m, m), axis = 1), [[1,2,3,1,2,3],[11,12,13,11,12,13]])
def testDiagonal (self): "Test the diagonal function." b=MA.array([[1,2,3,4],[5,6,7,8]]*2) assert eq(MA.diagonal(b), [1,6,3,8]) assert eq(MA.diagonal(b, -1), [5,2,7]) c = MA.array([b,b]) assert eq(MA.diagonal(c,1), [[2,7,4], [2,7,4]])
def _subSampleByAveraging(self, var, timeVar, flagVar, samplingRate, flagsToUse): """ Returns a new variable which is 'var' sub-sampled by averaging at the given samplingRate with data selected according to flagVar including the flag values specified in flagsToUse (defaults are 0 and 1). """ maskedArray=self._getMaskedArray(var, flagVar, flagsToUse) shape=var.shape if shape[1]==1: newNumArray=MV.ravel(maskedArray) newArrayMask=MV.ravel(maskedArray.mask()) newArray=MA.masked_array(newNumArray, mask=newArrayMask, fill_value=maskedArray.fill_value()) else: newArray=Numeric.zeros(shape[0], 'f') for t0 in range(shape[0]): # Set as missing if less than half are valid t1Array=maskedArray[t0] if samplingRate==1: # If half or more are good values then calculate the mean if t1Array.count()>=(shape[1]/2.): newArray[t0]=MA.average(t1Array) # otherwise set as missing value else: newArray[t0]=maskedArray.fill_value() else: raise "Averaging for non 1Hz sampling rates not yet supported!" # Now re-construct variable axes etc newTimeAxis=self._flatten2DTimeAxis(timeVar, samplingRate) newVar=self._recreateVariable(var, newArray, newTimeAxis, flagVar, max(flagsToUse), missingValue=maskedArray.fill_value(), sampleBy="averaging") return newVar
def Init_diag(Ain,Aout,fignum): global Ptot,RatioM,Tmincrit,Cpa,Cpv,Lv,namincrit,Hmincrit,Tmaxcrit,Tminopt global Ratiofig,fig_hauteur,fig_longueur,RatioAxes,lineprops,lineprops_HR,x1,y1,x3,y3 Ptot=101325.0 MO=15.9994 MN=14.0067 MAr=39.948 Mair=0.78084*2*MN+0.20946*2*MO+0.009340*MAr MH=1.00794 MH2O=2*MH+MO RatioM=MH2O/Mair Tmincrit=-30.0 Cpa=Cpas(C2K(Tmincrit)) Cpv=Cpvap(C2K(0.1)) Lv=ChLv(C2K(0.1)) namincrit=RatioM*Pvap(Tmincrit)/(Ptot-Pvap(Tmincrit)) Hmincrit=Cpa*Tmincrit+namincrit*(Lv+Cpv*Tmincrit) Tmaxcrit=260.0 Tminopt=-30.0 Ratiofig=21.0/29.7 fig_hauteur=6 #inches fig_longueur=fig_hauteur/Ratiofig figure(fignum,figsize=(fig_longueur,fig_hauteur)) lineprops = dict(linewidth=0.5, color='gray', linestyle='-',antialiased=True) lineprops_HR = dict(linewidth=1, color='blue', linestyle='-',antialiased=True) Tinit=20.0 Cpa=Cpas(C2K(Tinit)) Cpv=Cpvap(C2K(Tinit)) Lv=ChLv(C2K(Tinit)) maskin=Ma.getmask(Ain.car) maskout=Ma.getmask(Aout.car) # Il s'agit ici de fixer les valeurs moyennes de Cpa,Cpv et Lv # en faisant le calcul de l'air moyen (i.e. au centre du diagramme # le calcul s'arrête à 1e-6 de différence sur Cpa,Cpv et Lv crit=True while crit: Ain.car=Ma.array(Ain.car,copy=0,mask=maskin) Aout.car=Ma.array(Aout.car,copy=0,mask=maskout) Fill_Air(Ain) Fill_Air(Aout) Amoy=air() Amoy.definir(["H","na"],[mean([Ain.car[0],Aout.car[0]]), mean([Ain.car[1],Aout.car[1]])]) Fill_Air(Amoy) Tmoy=Amoy.car[2] Cpa_new=Cpas(C2K(Tmoy)) Cpv_new=Cpvap(C2K(Tmoy)) Lv_new=ChLv(C2K(Tmoy)) crit=((Cpa-Cpa_new)**2+(Cpv-Cpv_new)**2+(Lv-Lv_new)**2)>1e-6 Cpa=Cpa_new Cpv=Cpv_new Lv=Lv_new LH,Lna,LT,LHR,Lair=Return_levels_HnaTHR(Ain,Aout) x1,y1=Transf_xy(Lna[0],Lair[0].car[0]) x3,y3=Transf_xy(Lna[-1],Lair[2].car[0]) RatioAxes=(y3-y1)/(x3-x1) return LH,Lna,LT,LHR,Lair
def testReshape (self): "test reshape" w = MA.reshape(self.m, (-1,2)) assert eq(w, [[1,2],[3,11],[12,13]]) w = MA.reshape (self.a, (2,3)) assert eq(w, [[0, 1, 2], [3, 4, 5]]) self.failUnlessRaises(ValueError, MA.reshape, MA.arange(5), (3,2)) self.failUnlessRaises(ValueError, MA.reshape, self.a, (4,-1)) self.failUnlessRaises(ValueError, MA.reshape, self.a, (5,6))
def _check(a): self.assertEqual(a['na'], na) self.assertEqual(a['maa'], maa) self.assertEqual(a['na'][500:-500], na[500:-500]) self.assertEqual(a['maa'][500:-500], maa[500:-500]) scatter = Numeric.array(xrange(0, len(na), 3)) self.assertEqual(a['na'].take(scatter), Numeric.take(na, scatter)) self.assertEqual(Numeric.take(a['na'],scatter), Numeric.take(na, scatter)) self.assertEqual(MA.take(a['maa'], scatter), MA.take(maa, scatter))
def testLogical (self): "Test logical_and, logical_or, sometrue, alltrue" x = MA.array([1,1,0,0]) y = MA.array([1,0,1,0]) assert eq(MA.logical_and (x,y), [1,0,0,0]) assert eq(MA.logical_or (x,y), [1,1,1,0]) assert MA.sometrue(x) assert not MA.sometrue(MA.zeros((3,))) assert MA.alltrue(MA.ones((3,))) assert not MA.alltrue(x)
def testZeros(self): "Test zeros" y = MA.zeros((2,3)) assert y.shape == (2,3) assert y.typecode() == MA.Int assert eq(y, 0) z = MA.zeros((2,3), MA.Float) assert z.shape == (2,3) assert eq (y, z) self.failUnlessRaises(ValueError, MA.zeros, (-5,))
def testArange(self): "Test arange" x = MA.arange(5) assert eq(x, [0,1,2,3,4]) x = MA.arange(4, 6) assert eq(x, [4,5]) x = MA.arange(0, 1.5, .5) assert eq (x, [0.0, 0.5, 1.0]) assert len(MA.arange(5,3,2)) == 0 x = MA.arange(3L) assert x[1] == 1L assert x.typecode() == MA.PyObject
def testTypecodeSpecifying(self): 'Test construction using the type codes.' from Precision import typecodes thetypes = typecodes['Integer'] \ + typecodes['UnsignedInteger'] \ + typecodes['Float'] \ + typecodes['Complex'] for t in thetypes: x = MA.array([1,2,3], t) assert x.typecode() == t x = MA.array(['hi', 'hoo'], 'c') assert x.typecode() == 'c'
def testOnes(self): "Test ones" y = MA.ones((2,3)) assert y.shape == (2,3) assert y.typecode() == MA.Int assert eq(y.flat, 1) z = MA.ones((2,3), MA.Float) assert z.shape == (2,3) assert eq(y, z) w = MA.ones((2,3), MA.Int16) assert eq(w, MA.array([[1,1,1],[1,1,1]],'s')) self.failUnlessRaises(ValueError, MA.ones, (-5,))
def getMAMid(MA_Short, MA_Mid, symbol, K_MIN, indexoffset=0): #计算ma以及ma_cross bardata = pd.read_csv(DC.BAR_DATA_PATH + symbol + '\\' + symbol + ' ' + str(K_MIN) + '.csv') madf = pd.DataFrame({'close': bardata.close}) madf['utc_time'] = bardata['utc_time'] madf['strtime'] = bardata['strtime'] madf['index'] = bardata['Unnamed: 0'] - indexoffset #madf['MA_Short'] = MA.calMA(madf['close'], MA_Short) madf['MA_Mid'] = MA.calMA(madf['close'], MA_Mid) madf['MA_True'], madf['MA_Cross'] = MA.dfCross(madf, 'close', 'MA_Mid') #madf.to_csv("mamid.csv") return madf
def testIndexing (self): 'Test indexing operations.' x = MA.array([0,1,2,3,4,5]) for i in range(len(x)): assert i == x[i] x[2] = 20 assert eq(x, x[...]) w = MA.array([None]) assert w.typecode() == MA.PyObject assert w[0] is None assert isinstance(x[2], types.IntType) assert x[2] == 20 x = MA.array([0,1,2,3,4,5,6]) assert eq (x[4:1:-1], [4,3,2]) assert eq(x[4:1:-2], [4,2]) assert eq(x[::-1], [6, 5,4,3,2,1,0]) assert eq(x[2:-1], [2,3,4,5]) m = MA.array([[1,2,3],[11,12,13]]) assert m[0,2] == 3 assert isinstance(m[0,2], types.IntType) assert eq(m[...,1], [2,12]) assert eq(MA.arange(6)[..., MA.NewAxis], [[0],[1],[2],[3],[4],[5]]) x = MA.array([1,2,3]) y = MA.array(x) x[0] == 66 assert y[0] != 66 b=MA.array([[1,2,3,4],[5,6,7,8]]*2) # assert b[1:1].shape == (0,4) # assert b[1:1, :].shape == (0,4) # assert b[10:].shape == (0,4) assert eq(b[2:10], [[1,2,3,4],[5,6,7,8]]) assert eq(b[2:10, ...], [[1,2,3,4],[5,6,7,8]])
def copyTest (self): "Test how MA works with the copy module." import copy x = MA.array([1,2,3]) y = [1, x, 3] c1 = copy.copy(x) assert MA.allclose(x,c1) x[1] = 4 assert not MA.allclose(x,c1) c2 = copy.copy(y) assert id(c2[1]) == id(x) c3 = copy.deepcopy(y) assert id(c3[1]) != id(x) assert MA.allclose(c3[1], x)
def derive_format_str(self, data): if len(data) == 0: return self.default_format_str width = 10 max = MA.maximum(MA.absolute(data)) if max > 0: decimals = width - math.log(max, 10) - 2 if decimals < 0: decimals = 0 elif max >= 0.00001: decimals = 8 else: return '%10.10g' return '%%%d.%df' % (width, decimals)
def _getMaskedArray(self, var, flagVar=None, flagValues=(0,1), missingValuesToTest=(-9999., -32767.)): """ Returns a masked array that has the values only present where the flag values are acceptable and the data itself is not missing. """ if flagVar: flagFillValue=-1 flagGTValue=max(flagValues) flagMaskWhereUnacceptable=MA.masked_greater(flagVar, flagGTValue).mask() flagMaskWhereMissing=MA.masked_equal(flagVar, flagFillValue).mask() flagMask=flagMaskWhereUnacceptable+flagMaskWhereMissing for fv in missingValuesToTest: if fv in MV.ravel(var): print "Setting missing value for '%s' as: %s" % (var.id, fv) varFillValue=fv else: varFillValue=missingValuesToTest[0] if flagVar: varMask=MA.masked_array(var, mask=MA.equal(var, varFillValue), fill_value=varFillValue).mask() fullmask=MA.bitwise_or(varMask, flagMask) maskedArray=MA.masked_array(var, mask=fullmask, fill_value=varFillValue) else: maskedArray=MA.masked_array(var, mask=MA.equal(var, varFillValue), fill_value=varFillValue) #maskedArray=cdms.createVariable(maskedArray, id=var.id, fill_value=varFillValue) #maskedArray.missing_value=varFillValue return maskedArray
def testPickle (self): "Test pickling of MA arrays." import pickle x = MA.arange(10) fpik = open('foo.pik', 'wb') pickle.dump(x, fpik, 0) fpik.close() fpik = open('foo.pik', 'rb') y = pickle.load(open('foo.pik', 'rb')) fpik.close() assert eq(y,[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) assert MA.alltrue(MA.equal(x,y)) assert MA.sometrue(MA.equal(x,3)) assert y.shape == (10,)
def _getMaskedArray(self, var, flagVar=None, flagValues=(0, 1), missingValuesToTest=(-9999.0, -32767.0)): """ Returns a masked array that has the values only present where the flag values are acceptable and the data itself is not missing. """ if flagVar: flagFillValue = -1 flagGTValue = max(flagValues) flagMaskWhereUnacceptable = MA.masked_greater(flagVar, flagGTValue).mask() flagMaskWhereMissing = MA.masked_equal(flagVar, flagFillValue).mask() flagMask = flagMaskWhereUnacceptable + flagMaskWhereMissing for fv in missingValuesToTest: if fv in MV.ravel(var): print "Setting missing value for '%s' as: %s" % (var.id, fv) varFillValue = fv else: varFillValue = missingValuesToTest[0] if flagVar: varMask = MA.masked_array(var, mask=MA.equal(var, varFillValue), fill_value=varFillValue).mask() fullmask = MA.bitwise_or(varMask, flagMask) maskedArray = MA.masked_array(var, mask=fullmask, fill_value=varFillValue) else: maskedArray = MA.masked_array(var, mask=MA.equal(var, varFillValue), fill_value=varFillValue) # maskedArray=cdms.createVariable(maskedArray, id=var.id, fill_value=varFillValue) # maskedArray.missing_value=varFillValue return maskedArray
def testStandard (self): "Test sqrt, log, log10, exp" assert eq (MA.exp(1.0), MA.e) assert eq (MA.sqrt(self.a), map(math.sqrt, self.a)) assert eq (MA.log(self.a), map(math.log, self.a)) assert eq (MA.log10(self.a), map(math.log10, self.a)) assert eq (MA.exp(self.a), map(math.exp, self.a)) assert eq (MA.sqrt(self.m).flat, map(math.sqrt, self.m.flat)) assert eq (MA.log(self.m).flat, map(math.log, self.m.flat)) assert eq (MA.log10(self.m).flat, map(math.log10, self.m.flat)) assert eq (MA.exp(self.m).flat, map(math.exp, self.m.flat))
def testReductions (self): "Tests of reduce attribute." a = MA.arange(6) m = MA.array([[1,2,3],[11,12,13]]) assert MA.add.reduce(a) == 15 assert MA.multiply.reduce(m.shape) == len(m.flat) assert eq(MA.add.reduce (m, 0), [12,14,16]) assert eq(MA.add.reduce (m, -1), [6,36]) assert eq(MA.multiply.reduce (m, 0), [11,24,39]) assert eq(MA.multiply.reduce (m, -1), [6,11*12*13]) assert MA.add.reduce([1]) == 1 assert MA.add.reduce([]) == 0 assert MA.multiply.reduce([]) == 1 assert MA.minimum.reduce(a) == 0 assert MA.maximum.reduce(a) == 5
def testSpacesaver (self): "Test the spacesaver property (Travis Oliphant)" # Test of savespace property: Travis Oliphant a = MA.array([1,2,3,4],savespace=1) assert a.spacesaver() self.assertEqual(a.typecode(), 's') b = MA.array(a,'f') self.assertEqual(b.typecode(), 'f') assert b.spacesaver() a.savespace(0) assert not a.spacesaver() assert b.spacesaver() d = 4 * b assert b.typecode() == d.typecode() self.failUnlessRaises, TypeError, MA.arccos, (b/10.0)
def take(self, data, want): if type(data) is MA.MaskedArray: return MA.take(data, want) elif type(data) is Numeric.ArrayType: return Numeric.take(data, want) else: return _BaseDataType.take(self, data, want)
def getNewLonAxis(oldLonAxis, startLon=-180): assert (oldLonAxis.isLongitude()) oldLonVals = oldLonAxis.getData() newLonVals = (oldLonVals - startLon ) % 360 + startLon # range "startLon" to "startLon + 360" lonValShift = newLonVals - oldLonVals nlon = len(newLonVals) ## ## identify shift by jump in longitude values for i in range(1, nlon): if newLonVals[i] < newLonVals[i - 1]: nShift = i break else: nShift = 0 ## create array with new longitude values newLonVals = MA.concatenate((newLonVals[nShift:], newLonVals[:nShift])) ## print oldLonVals ## print newLonVals newLonAxis = cdms.createAxis(newLonVals) newLonAxis.designateLongitude() newLonAxis.replace_external_attributes(oldLonAxis.attributes) # return newLonAxis, nShift, lonValShift
def get_store_chain(self, data, mask=None): """ Return a chain of generators for processing the src data, which can either be a list or an iterable. """ multisourcecols = self.multisourcecols or self.heterosourcecols if multisourcecols: src = self._multisrc_gen(multisourcecols) else: src = iter(data) if MA.isMaskedArray(data) and mask is None: mask = data.mask() if mask is not None: src = self._mask_gen(src, mask) if self.is_multivalue(): src = self._tuple_gen(src, self.ignorenone) if self.missingvalues: src = self._tuplemissing_gen(src, self.missingvalues) else: if self.calculatedby: src = self._calcby_gen(src, self.calculatedby, self.calculatedargs) if self.missingvalues: src = self._missing_gen(src, self.missingvalues) src = self._storedata_gen(src) return src
def testComplex (self): "Test complex numbers" y = MA.arange(5) * 1.0 x = y + y[::-1] * 1.0j assert x.typecode() == MA.Complex assert eq(x.real, y) assert eq(x.imag, y[::-1])
def to_R(self, ds, dates_as_factor, **kwargs): col = ds[self.colname] if MA.isMaskedArray(col.data): if Numeric.alltrue(col.data.mask()): raise PlotError('%r: all data-points masked' % col.name) return rconv.MA_to_R(col.data) elif col.is_datetimetype(): return rconv.Missing_Date_to_R(col.data) else: return col.data
def _subSampleByAveraging(self, var, timeVar, flagVar, samplingRate, flagsToUse): """ Returns a new variable which is 'var' sub-sampled by averaging at the given samplingRate with data selected according to flagVar including the flag values specified in flagsToUse (defaults are 0 and 1). """ maskedArray = self._getMaskedArray(var, flagVar, flagsToUse) shape = var.shape if shape[1] == 1: newNumArray = MV.ravel(maskedArray) newArrayMask = MV.ravel(maskedArray.mask()) newArray = MA.masked_array(newNumArray, mask=newArrayMask, fill_value=maskedArray.fill_value()) else: newArray = Numeric.zeros(shape[0], "f") for t0 in range(shape[0]): # Set as missing if less than half are valid t1Array = maskedArray[t0] if samplingRate == 1: # If half or more are good values then calculate the mean if t1Array.count() >= (shape[1] / 2.0): newArray[t0] = MA.average(t1Array) # otherwise set as missing value else: newArray[t0] = maskedArray.fill_value() else: raise "Averaging for non 1Hz sampling rates not yet supported!" # Now re-construct variable axes etc newTimeAxis = self._flatten2DTimeAxis(timeVar, samplingRate) newVar = self._recreateVariable( var, newArray, newTimeAxis, flagVar, max(flagsToUse), missingValue=maskedArray.fill_value(), sampleBy="averaging", ) return newVar
def to_frame(self, frame, ds, **kwargs): limit_cols = self.limit_cols(ds) if limit_cols: self.max = 0 self.min = 0 for colname in (self.colname, ) + limit_cols: col = ds[colname] if MA.isMaskedArray(col.data): if Numeric.alltrue(col.data.mask()): raise PlotError('%r: all data-points masked' % col.name) data = rconv.MA_to_R(col.data) self.max = max(MA.maximum(col.data), self.max) self.min = min(MA.minimum(col.data), self.min) else: data = col.data self.max = max(max(col.data), self.max) self.min = min(min(col.data), self.min) frame[rconv.rname(colname)] = data else: super(RMeasureCondCol, self).to_frame(frame, ds, **kwargs)
def from_col(cls, col, values=None): self = cls(col.name, col.label, col=col) if values is not None: # this_values = sets.Set(col.inverted.keys()) # wanted_values = sets.Set(values) # if not wanted_values.issubset(this_values): # missing_vals = [repr(v) for v in (wanted_values - this_values)] # raise Error('column %r values mismatch - %r' % # (self.name, ', '.join(missing_vals))) self.values = values else: self.values = col.inverted.keys() self.values.sort() self.indices = fully_masked(len(col)) if 0: # AM - the .inverted attribute of filtered datasets is returning # invalid row indices, and the summ code relies on these indices, # so the following code cannot be used until they are fixed. self.axis_map = {} for i, v in enumerate(self.values): self.axis_map[v] = i vec = col.inverted.get(v) if vec is not None: for idx in vec: try: self.indices[idx] = i except IndexError: print col.name, idx raise MA.put(self.indices, vec, i) else: axis_map = dict([(v, i) for i, v in enumerate(self.values)]) for i, v in enumerate(col.data): try: self.indices[i] = axis_map[v] except KeyError: pass self.axis_map = axis_map return self
def __setitem__(self, key, a): index = self.dict.get(key) if index is None: if MA.isMaskedArray(a): index = self._save_new_masked(a) elif type(a) == Numeric.ArrayType: index = self._save_new_array(a) elif type(a) == types.StringType: index = self._save_new_str(a) else: index = self._save_new_str(repr(a)) self.dict[key] = index self.dict_dirty = 1 else: if MA.isMaskedArray(a): self._save_masked(index, a) elif type(a) == Numeric.ArrayType: self._save_array(index, a) elif type(a) == types.StringType: self._save_str(index, a) else: index = self._save_str(index, repr(a))
def replicate_axis(data, axis, size): """ Returns a new array with an additional axis of the specified size. The new axis is formed by replicating along the other axes. """ assert axis > 0 repeat_map = size * Numeric.ones(data.shape[axis-1]) shape = list(data.shape) data = MA.repeat(data, repeat_map, axis=axis-1) shape.insert(axis, size) data.shape = tuple(shape) return data
def __getitem__(self, key): index = self.dict[key] blob = self.store[index] if blob.type == BLOB_ARRAY: return MmapArray(blob) elif blob.type == BLOB_FILLED: data = MmapArray(blob) blob = self.store[blob.other] mask = MmapArray(blob) return MA.array(data, mask=mask) elif blob.type == BLOB_STRING: return blob.as_str() else: raise Error('bad BLOB type %s in index' % blob.type)
def BOLL(df, N=26, M=26, P=2): print 'start computing DMI' mid = MA.calMA(df['close'], N) tmp2 = pd.rolling_std(df['close'], M) TOP = mid + P * tmp2 BOTTOM = mid - P * tmp2 df1 = pd.DataFrame({ 'close': df['close'], 'MID': mid, 'TOP': TOP, 'BOTTOM': BOTTOM }) print 'DMI compute finished' return df1
def affiche(self) : l=[] for i in range(6) : if Ma.getmask(self.car)[i] : l.append("--") else : if i==0 : s="%0.f" %(self.car[i]) elif i==1 : s="%0.3f" %(self.car[i]) else : s="%0.2f" %(self.car[i]) l.append(s) return "[ "+join(l," , ")+" ]"
def __getitem__(self, key): index = self.dict[key] blob = self.store[index] if blob.type == BLOB_ARRAY: return MmapArray(blob) elif blob.type == BLOB_FILLED: data = MmapArray(blob) blob = self.store[blob.other] mask = MmapArray(blob) return MA.array(data, mask = mask) elif blob.type == BLOB_STRING: return blob.as_str() else: raise Error('bad BLOB type %s in index' % blob.type)
def dataPrepare(self): startTime = datetime.time(self.tradeEndHour, self.tradeStartMin, 0).strftime("%H:%M:%S") if self.mode == 4: d, t = self.start_time.split(' ', 1) y, m, d = d.split('-', 2) d = datetime.date(int(y), int(m), int(d)) startDate = ( d - datetime.timedelta(days=self.backwardDay)).strftime("%Y-%m-%d") endTime = self.start_time else: startDate = ( datetime.date.today() - datetime.timedelta(days=self.backwardDay)).strftime("%Y-%m-%d") endTime = datetime.date.today().strftime( "%Y-%m-%d") + ' ' + startTime sT = startDate + ' ' + startTime bars = self.get_bars(self.exchange_id + '.' + self.sec_id, self.K_min * 60, sT, endTime) #这里数据只用来计算MA rownum = 0 for bar in bars: rownum = self.update_dailyBar(bar) if rownum % self.K_min == 0 and rownum >= self.K_min: self.update_dailyBarMin(bar) self.MAdf = MA.MA(self.dailyBarMin.close, self.MA_Short, self.MA_Long) self.DMIdf = DMI.DMI(self.dailyBarMin, self.DMI_N, self.DMI_M) kdjdf = KDJ.KDJ(self.dailyBarMin, self.KDJ_N, self.KDJ_M) kdjdf['KDJ_True'] = 0 kdjdf.loc[(self.KDJ_HLim > kdjdf['KDJ_K']) & (kdjdf['KDJ_K'] > kdjdf['KDJ_D']), 'KDJ_True'] = 1 kdjdf.loc[(self.KDJ_LLim < kdjdf['KDJ_K']) & (kdjdf['KDJ_K'] < kdjdf['KDJ_D']), 'KDJ_True'] = -1 self.KDJdf = kdjdf #下面要再做实盘下当天数据的处理 if self.mode == 2: pass if rownum > 0: self.last_update_time = datetime.datetime.fromtimestamp( self.dailyBar.iloc[-1].utctime) print( "------------------------data prepared-----------------------------" ) pass
def store_data(self, data, mask, filename=None): if mask is None: data = Numeric.array(data, typecode=self.numeric_type) else: data = MA.array(data, typecode=self.numeric_type, mask=mask) if filename: try: os.unlink(filename) except OSError: pass data_blob = ArrayDict(filename, 'w+') data_blob['data'] = data del data_blob # this writes the data to disc - # we really need a .sync() method... return None # Flag for load on demand else: return data
def calDMI(df, N=14, M=6): high = df.high low = df.low close = df.close closeshift1 = close.shift(1).fillna(0) open = df.open c = high - low d = high - closeshift1 df1 = pd.DataFrame({'c': c, 'd': d}) df1['A'] = df1.max(axis=1) df1.drop('c', axis=1, inplace=True) df1.drop('d', axis=1, inplace=True) df1['B'] = np.abs(low - closeshift1) df1['C'] = df1.max(axis=1) # df1.drop('A',axis=1,inplace=True) # df1.drop('B',axis=1,inplace=True) df1['TR'] = df1['C'].rolling(N).sum() # 2、HD=最高价-昨日最高价 # 3、LD=昨日最低价-最低价 HD = high - high.shift(1).fillna(0) LD = low.shift(1).fillna(0) - low df1['HD'] = HD df1['LD'] = LD # DMP:= SUM(IFELSE(HD>0 && HD>LD,HD,0),N); # DMM:= SUM(IFELSE(LD>0 && LD>HD,LD,0),N); df2 = pd.DataFrame({'HD': HD, 'LD': LD}) df2['DMP_1'] = df2[(df2['HD'] > df2['LD']) & (df2['HD'] > 0)]['HD'] df2['DMM_1'] = df2[(df2['LD'] > df2['HD']) & (df2['LD'] > 0)]['LD'] df2 = df2.fillna(0) df1['DMP'] = df2['DMP_1'].rolling(N).sum() df1['DMM'] = df2['DMM_1'].rolling(N).sum() del df2 # 6、PDI=DMP*100/TR # 7、MDI=DMM*100/TR df1['PDI'] = df1['DMP'] * 100 / df1['TR'] df1['MDI'] = df1['DMM'] * 100 / df1['TR'] # ADX: MA(ABS(MDI-PDI)/(MDI+PDI)*100,M); # ADXR:(ADX+REF(ADX,M))/2; df1['ADX'] = MA.calMA( np.abs(df1['MDI'] - df1['PDI']) / (df1['MDI'] + df1['PDI']) * 100, M) df1['ADXR'] = (df1['ADX'] + df1['ADX'].shift(M).fillna(0)) / 2 return df1['PDI'], df1['MDI'], df1['ADX'], df1['ADXR']
def train(args, model): model.train() loader = DataLoader(MA( '/Users/zhangweidong03/Code/dl/pytorch/github/piwise/MAdata', input_transform, target_transform), num_workers=1, batch_size=1, shuffle=True) weight = torch.ones(2) weight[0] = 0 use_cuda = False if use_cuda: criterion = CrossEntropyLoss2d(weight.cuda()) else: criterion = CrossEntropyLoss2d(weight) criterion = CrossEntropyLoss2d() optimizer = SGD(model.parameters(), 1e-4, .9, 2e-5) for epoch in range(1, 51): epoch_loss = [] for step, (images, labels) in enumerate(loader): if use_cuda: images = images.cuda() labels = labels.cuda() inputs = Variable(images) targets = Variable(labels) outputs = model(inputs) optimizer.zero_grad() loss = criterion(outputs, targets[:, 0]) loss.backward() optimizer.step() epoch_loss.append(loss.data[0]) average = sum(epoch_loss) / len(epoch_loss) print(f'loss: {average} (epoch: {epoch}, step: {step})')
def updateParadata(self): #更新MA数据 self.MAdf.loc[self.MAdf.shape[0]] = MA.newMA(self.dailyBarMin.close, self.MAdf, self.MA_Short, self.MA_Long) #更新DMI数据 self.DMIdf.loc[self.DMIdf.shape[0]] = DMI.newDMI( self.dailyBarMin, self.DMIdf, self.DMI_N, self.DMI_M) #更新KDJ数据 kdj = KDJ.newKDJ(self.dailyBarMin, self.KDJdf, self.KDJ_N, self.KDJ_M) newk = kdj[0] newd = kdj[1] KDJ_True = 0 if self.KDJ_HLim > newk and newk > newd: KDJ_True = 1 elif self.KDJ_LLim < newk and newk < newd: KDJ_True = -1 kdj.append(KDJ_True) self.KDJdf.loc[self.KDJdf.shape[0]] = kdj pass
def summ_to_array(ds, measure, debug=False): from SOOMv0.CrossTab import CrossTab ct = CrossTab.from_summset(ds) data = ct[measure].data if MA.isMaskedArray(data): data = data.filled() # Dealing with frequencies, so this is valid. data = data.astype(Numeric.Float64) if debug: print type(data) if debug: print data array = r.as_array(data) dimnames = [] labels = [] for axis in ct.axes: dimnames.append([axis.col.do_outtrans(v) for v in axis.values]) labels.append(axis.col.label or axis.col.name) dimnames = r.list(*dimnames) dimnames = r.names__(dimnames, labels) array = r.dimnames__(array, dimnames) if debug: r.print_(array) return array
def test_arraydict(self): def _check(a): self.assertEqual(a['na'], na) self.assertEqual(a['maa'], maa) self.assertEqual(a['na'][500:-500], na[500:-500]) self.assertEqual(a['maa'][500:-500], maa[500:-500]) scatter = Numeric.array(xrange(0, len(na), 3)) self.assertEqual(a['na'].take(scatter), Numeric.take(na, scatter)) self.assertEqual(Numeric.take(a['na'],scatter), Numeric.take(na, scatter)) self.assertEqual(MA.take(a['maa'], scatter), MA.take(maa, scatter)) tmpfile = TempFile('soomtestarray_tmpfile') try: a = soomarray.ArrayDict(tmpfile.fn(), 'w') self.assertEqual(len(a), 0) self.assertRaises(KeyError, a.__getitem__, 'not_found') na = Numeric.arrayrange(10000) a['na'] = na na[1000] = -2222 # MmapArray __setitem__ a['na'][1000] = -2222 maa = MA.arrayrange(10000) for i in xrange(0, len(maa), 5): maa[i] = MA.masked a['maa'] = maa maa[1000] = -2222 a['maa'][1000] = -2222 _check(a) del a a = soomarray.ArrayDict(tmpfile.fn(), 'r') _check(a) finally: try: del a except UnboundLocalError: pass tmpfile.done()
def whos(module=globals().keys(), what='A', sstr='', casesen=0, \ out=0): #===================================== ''' # Before using whos, do: execfile('whos.py') #=== EXAMPLES === # to see a list of all objects in the global scope whos() # to see a list of functions whos(what='f') # to see a list of variables (also their values) whos(what='v') # to see a list of Numeric array whos(what='a') # to see a list of variables in module os import os whos(os, what='v') # to see a list of all variables of a dictionary whos({}, what='v') # to see a list of all functions of a list whos([], what='f') # to see a list of variables whose name contains 'path' whos(what='v', sstr='path') # to see a list of variables whose name contains 'path' (case insensitive) whos(what='v', sstr='path', casesen=0) Argument what can be one of the following: A -- all a -- array f -- function m -- module v -- variable ''' whosList = 0 if type(module) == type([]): if len(module) > 0: dirs0 = module whosList = 1 if not whosList: temp2 = dir(module) dirs0 = map(lambda a: 'module.' + a, temp2) type2 = {} arrayOk = 1 if what == 'a' or what == 'A': try: import Numeric as Num temp1 = type(Num.zeros((1, ))) type2['array'] = temp1 except: arrayOk = 0 if arrayOk: import MA temp1 = type(MA.zeros((1, ))) type2['MA'] = temp1 if what == 'a' and not arrayOk: print 'Cannot import Numeric' return [] if what == 'v' or what == 'A': temp1 = type(1) type2['int'] = temp1 temp1 = type(1L) type2['long'] = temp1 temp1 = type(1.1) type2['float'] = temp1 temp1 = type('a') type2['str'] = temp1 if what == 'f' or what == 'A': temp1 = type(lambda a: a) type2['function'] = temp1 if what == 'm' or what == 'A': import os temp1 = type(os) type2['module'] = temp1 type1 = type2.values() dirs0.sort() if sstr: if casesen: p1 = re.compile(sstr) else: p1 = re.compile(sstr, re.I) dirs1 = [] for d in dirs0: if p1.search(d): dirs1.append(d) dirs0 = dirs1 collect1 = [] dirsType = [] if what == 'A': for i in range(len(dirs0)): t1 = eval('type(' + dirs0[i] + ')') collect1.append(i) dirsType.append(t1) else: for i in range(len(dirs0)): t1 = eval('type(' + dirs0[i] + ')') #print t1, dirs0[i] if t1 in type1: collect1.append(i) dirsType.append(t1) for i in range(len(collect1)): ii = collect1[i] iType = dirsType[i] iType1 = string.split(str(iType), "'")[1] if iType1[:2] == "MA": iType1 = iType1[:2] if iType1[:7] == "builtin": iType1 = iType1[:7] print '%15s %15s' % (iType1, string.split(dirs0[ii], '.')[-1]), if (what == 'a' or what == 'A') and arrayOk: if iType == type2['MA'] or iType == type2['array']: print '%s %20s ' % (eval(dirs0[ii] + '.typecode()'), str(eval(dirs0[ii] + '.shape'))), if iType == type2['MA']: print 'unmask=%d of %d' % (eval(dirs0[ii] + '.count()'), eval(dirs0[ii] + '.size()')), if what == 'v' or what == 'A': if iType == type2['int'] or iType == type2[ 'long'] or iType == type2['float']: print '= ', eval(dirs0[ii]), if iType == type2['str']: temp3 = eval(dirs0[ii]) temp3 = string.replace(temp3, '\n', '\\') if len(temp3) < 30: print '= ', temp3, else: print '= ', temp3[:30] + ' ...', print '' if out: return [dirs0[i] for i in collect1]
def train(args, model): model.train() # loader = DataLoader(MA('/Users/zhangweidong03/Code/dl/pytorch/github/piwise/MAdata', input_transform, target_transform), # num_workers=1, batch_size=1, shuffle=True) loader = DataLoader(MA(args.datadir, input_transform, target_transform), num_workers=args.num_workers, batch_size=args.batch_size, shuffle=True) weight = torch.ones(2) weight[0] = 0 use_cuda = False if args.cuda: criterion = CrossEntropyLoss2d(weight.cuda()) else: criterion = CrossEntropyLoss2d(weight) criterion = CrossEntropyLoss2d() # optimizer = SGD(model.parameters(), 1e-4, .9, 2e-5) optimizer = Adam(model.parameters()) if args.model.startswith('FCN'): optimizer = SGD(model.parameters(), 1e-4, .9, 2e-5) if args.model.startswith('PSP'): optimizer = SGD(model.parameters(), 1e-2, .9, 1e-4) if args.model.startswith('Seg'): optimizer = SGD(model.parameters(), 1e-3, .9) # for epoch in range(1, 51): # epoch_loss = [] # # for step, (images, labels) in enumerate(loader): # if use_cuda: # images = images.cuda() # labels = labels.cuda() # # inputs = Variable(images) # targets = Variable(labels) # outputs = model(inputs) # # optimizer.zero_grad() # loss = criterion(outputs, targets[:, 0]) # loss.backward() # optimizer.step() # # epoch_loss.append(loss.data[0]) # # average = sum(epoch_loss) / len(epoch_loss) # print(f'loss: {average} (epoch: {epoch}, step: {step})') if args.steps_plot > 0: board = Dashboard(args.port) for epoch in range(1, args.num_epochs + 1): epoch_loss = [] for step, (images, labels) in enumerate(loader): if args.cuda: images = images.cuda() labels = labels.cuda() inputs = Variable(images) targets = Variable(labels) outputs = model(inputs) optimizer.zero_grad() loss = criterion(outputs, targets[:, 0]) loss.backward() optimizer.step() epoch_loss.append(loss.data[0]) if args.steps_plot > 0 and step % args.steps_plot == 0: image = inputs[0].cpu().data image[0] = image[0] * .229 + .485 image[1] = image[1] * .224 + .456 image[2] = image[2] * .225 + .406 board.image(image, f'input (epoch: {epoch}, step: {step})') board.image(color_transform(outputs[0].cpu().max(0)[1].data), f'output (epoch: {epoch}, step: {step})') board.image(color_transform(targets[0].cpu().data), f'target (epoch: {epoch}, step: {step})') if args.steps_loss > 0 and step % args.steps_loss == 0: average = sum(epoch_loss) / len(epoch_loss) print(f'loss: {average} (epoch: {epoch}, step: {step})') if args.steps_save > 0 and step % args.steps_save == 0: filename = f'{args.model}-{epoch:03}-{step:04}.pth' torch.save(model.state_dict(), filename) print(f'save: {filename} (epoch: {epoch}, step: {step})')
def IntradayFryWin(symbolinfo, rawdata, paraSet): setname = paraSet['Setname'] MACD_S = paraSet['MACD_S'] MACD_L = paraSet['MACD_L'] MACD_M = paraSet['MACD_M'] MA_N = paraSet['MA_N'] # print setname rawdata['Unnamed: 0'] = range(rawdata.shape[0]) #beginindex = rawdata.ix[0, 'Unnamed: 0'] # 计算MACD macd = MA.calMACD(rawdata['close'], MACD_S, MACD_L, MACD_M) rawdata['DIF'] = macd[0] rawdata['DEA'] = macd[1] rawdata['MA'] = MA.calEMA(rawdata['close'], MA_N) # 计算MACD的金叉和死叉 rawdata['MACD_True'], rawdata['MACD_Cross'] = MA.dfCross(rawdata, 'DIF', 'DEA') # ================================ 找出买卖点================================================ # 1.先找出SAR金叉的买卖点 # 2.找到结合判决条件的买点 # 3.从MA买点中滤出真实买卖点 # 取出金叉点 goldcrosslist = pd.DataFrame({'goldcrosstime': rawdata.loc[rawdata['MACD_Cross'] == 1, 'strtime']}) goldcrosslist['goldcrossutc'] = rawdata.loc[rawdata['MACD_Cross'] == 1, 'utc_time'] goldcrosslist['goldcrossindex'] = rawdata.loc[rawdata['MACD_Cross'] == 1, 'Unnamed: 0'] goldcrosslist['goldcrossprice'] = rawdata.loc[rawdata['MACD_Cross'] == 1, 'close'] # 取出死叉点 deathcrosslist = pd.DataFrame({'deathcrosstime': rawdata.loc[rawdata['MACD_Cross'] == -1, 'strtime']}) deathcrosslist['deathcrossutc'] = rawdata.loc[rawdata['MACD_Cross'] == -1, 'utc_time'] deathcrosslist['deathcrossindex'] = rawdata.loc[rawdata['MACD_Cross'] == -1, 'Unnamed: 0'] deathcrosslist['deathcrossprice'] = rawdata.loc[rawdata['MACD_Cross'] == -1, 'close'] goldcrosslist = goldcrosslist.reset_index(drop=True) deathcrosslist = deathcrosslist.reset_index(drop=True) # 生成多仓序列(金叉在前,死叉在后) if goldcrosslist.ix[0, 'goldcrossindex'] < deathcrosslist.ix[0, 'deathcrossindex']: longcrosslist = pd.concat([goldcrosslist, deathcrosslist], axis=1) else: # 如果第一个死叉的序号在金叉前,则要将死叉往上移1格 longcrosslist = pd.concat([goldcrosslist, deathcrosslist.shift(-1).fillna(0)], axis=1) longcrosslist = longcrosslist.set_index(pd.Index(longcrosslist['goldcrossindex']), drop=True) # 生成空仓序列(死叉在前,金叉在后) if deathcrosslist.ix[0, 'deathcrossindex'] < goldcrosslist.ix[0, 'goldcrossindex']: shortcrosslist = pd.concat([deathcrosslist, goldcrosslist], axis=1) else: # 如果第一个金叉的序号在死叉前,则要将金叉往上移1格 shortcrosslist = pd.concat([deathcrosslist, goldcrosslist.shift(-1).fillna(0)], axis=1) shortcrosslist = shortcrosslist.set_index(pd.Index(shortcrosslist['deathcrossindex']), drop=True) # 取出开多序号和开空序号 openlongindex = rawdata.loc[ (rawdata['MACD_Cross'] == 1) & (rawdata['close'] > rawdata['MA']) & (rawdata['ATR'] > 6)].index openshortindex = rawdata.loc[ (rawdata['MACD_Cross'] == -1) & (rawdata['close'] < rawdata['MA']) & (rawdata['ATR'] > 6)].index # 从多仓序列中取出开多序号的内容,即为开多操作 longopr = longcrosslist.loc[openlongindex] longopr['tradetype'] = 1 longopr.rename(columns={'goldcrosstime': 'opentime', 'goldcrossutc': 'openutc', 'goldcrossindex': 'openindex', 'goldcrossprice': 'openprice', 'deathcrosstime': 'closetime', 'deathcrossutc': 'closeutc', 'deathcrossindex': 'closeindex', 'deathcrossprice': 'closeprice'}, inplace=True) # 从空仓序列中取出开空序号的内容,即为开空操作 shortopr = shortcrosslist.loc[openshortindex] shortopr['tradetype'] = -1 shortopr.rename(columns={'deathcrosstime': 'opentime', 'deathcrossutc': 'openutc', 'deathcrossindex': 'openindex', 'deathcrossprice': 'openprice', 'goldcrosstime': 'closetime', 'goldcrossutc': 'closeutc', 'goldcrossindex': 'closeindex', 'goldcrossprice': 'closeprice'}, inplace=True) # 结果分析 result = pd.concat([longopr, shortopr]) result = result.sort_index() result = result.reset_index(drop=True) # result.drop(result.shape[0] - 1, inplace=True) result = result.dropna() # 去掉跨合约的操作 # 使用单合约,不用再去掉跨合约 # result = removeContractSwap(result, contractswaplist) slip = symbolinfo.getSlip() result['ret'] = ((result['closeprice'] - result['openprice']) * result['tradetype']) - slip result['ret_r'] = result['ret'] / result['openprice'] results = {} ''' # 使用单合约,策略核心内不再计算结果 if calcResult: result['commission_fee'], result['per earn'], result['own cash'], result['hands'] = RS.calcResult(result, symbolinfo, initialCash, positionRatio) endcash = result['own cash'].iloc[-1] Annual = RS.annual_return(result) Sharpe = RS.sharpe_ratio(result) DrawBack = RS.max_drawback(result)[0] SR = RS.success_rate(result) max_single_loss_rate = abs(result['ret_r'].min()) results = { 'Setname':setname, 'opentimes': result.shape[0], 'end_cash': endcash, 'SR': SR, 'Annual': Annual, 'Sharpe': Sharpe, 'DrawBack': DrawBack, 'max_single_loss_rate': max_single_loss_rate } closeopr = result.loc[:, 'closetime':'tradetype'] return result, rawdata, closeopr, results ''' return result
def oldnonzero(a): return a.nonzero()[0] else: from Numeric import * import MA def gettypecode(x): return x.typecode() oldnonzero = nonzero import os.path import time if not with_numpy: #-- Set this to a more reasonable value MA.set_print_limit(10000) # --- Since gist is only loaded on PE0 in the parallel # --- environment, the parallel module must also be loaded in to check # --- whether this is in fact running in parallel. # --- This creates a logical lparallel which is true when running in parallel # --- and false when running serial. # --- This also creates a number of routines which are needed for parallel # --- data handling. These routines all have sensible default results when # --- not running in parallel. from ueparallel import * try: if me == 0 and sys.platform != 'mac': from gist import *
def LvyiWin(symbolinfo, rawdata, paraset): setname = paraset['Setname'] KDJ_N = paraset['KDJ_N'] KDJ_M = paraset['KDJ_M'] KDJ_HLim = paraset['KDJ_HLim'] KDJ_LLim = paraset['KDJ_LLim'] DMI_N = paraset['DMI_N'] DMI_M = paraset['DMI_M'] MA_Short = paraset['MA_Short'] MA_Long = paraset['MA_Long'] rawdata['Unnamed: 0'] = range(rawdata.shape[0]) beginindex = rawdata.ix[0, 'Unnamed: 0'] # 处理KDJ数据:KDJ_OPEN做为最终KDJ的触发信号 # KDJ_True=1:80>k>D # KDJ_True=-1:20<K<D df_KDJ = KDJ.KDJ(rawdata, N=KDJ_N, M=KDJ_M) df_KDJ['KDJ_True'] = 0 df_KDJ.loc[(KDJ_HLim > df_KDJ['KDJ_K']) & (df_KDJ['KDJ_K'] > df_KDJ['KDJ_D']), 'KDJ_True'] = 1 df_KDJ.loc[(KDJ_LLim < df_KDJ['KDJ_K']) & (df_KDJ['KDJ_K'] < df_KDJ['KDJ_D']), 'KDJ_True'] = -1 # 处理DMI数据:DMI_GOLD_CROSS做为DMI的触发信号 # DMI_True=1:and PDI>MDI # DMI_True=-1:and PDI<MDI df_DMI = DMI.DMI(rawdata, N=DMI_N, M=DMI_M) ''' #2017-10-26 10:44 DMI_True的计算,移到DMI模块中执行 df_DMI['DMI_True']=0 df_DMI.loc[df_DMI['PDI']> df_DMI['MDI'], 'DMI_True'] = 1 df_DMI.loc[df_DMI['PDI']< df_DMI['MDI'], 'DMI_True'] = -1 ''' # 处理MA数据:MA_Cross做为MA的触发信号 # MA20_True=1:close>MA20 # MA20_True=-1:close<MA20 df_MA = MA.MA(rawdata['close'], MA_Short, MA_Long) ''' #2017-10-26: MA的计算转移到MA模块中 df_MA=pd.DataFrame({'close':rawdata['close']}) df_MA['MA_Short']=MA.calMA(df_MA['close'],MA_Short) df_MA['MA_Long']=MA.calMA(df_MA['close'],MA_Long) df_MA['MA_True'],df_MA['MA_Cross']=MA.dfCross(df_MA,'MA_Short','MA_Long') ''' ''' #处理BOLL数据:BOLL_Cross做为BOLL平仓的触发信号 df_BOLL=BOLL.BOLL(rawdata,N=BOLL_N,M=BOLL_M,P=BOLL_P) df_BOLL['BOLL_True']=0 df_BOLL.loc[df_BOLL['close']< df_BOLL['BOTTOM'], 'BOLL_True'] = -1 df_BOLL.loc[df_BOLL['close']> df_BOLL['TOP'], 'BOLL_True'] = 1 #BOLL的金叉,BOLL_True从0变成1是金叉,从0变成-1是死叉 df_BOLL['BOLL_Cross']=0 df_BOLL['bigger1'] = df_BOLL['BOLL_True'].shift(1).fillna(0) df_BOLL.loc[(df_BOLL['BOLL_True']==1)&(df_BOLL['bigger1']==0), 'BOLL_Cross'] = 1 df_BOLL.loc[(df_BOLL['BOLL_True']==-1)&(df_BOLL['bigger1']==0), 'BOLL_Cross'] = -1 df_BOLL.drop('bigger1', axis=1, inplace=True) df_BOLL.to_csv('df_BOLL.csv') df_BOLL.drop('close',axis=1,inplace=True) ''' df_MA.drop('close', axis=1, inplace=True) df = pd.concat([rawdata, df_DMI, df_KDJ, df_MA], axis=1) # 找出买卖点: # 1.先找出MA金叉的买卖点 # 2.找到结合判决条件的买点 # 3.从MA买点中滤出真实买卖点 # 取出金叉点 goldcrosslist = pd.DataFrame({'goldcrosstime': df.loc[df['MA_Cross'] == 1, 'strtime']}) goldcrosslist['goldcrossutc'] = df.loc[df['MA_Cross'] == 1, 'utc_time'] goldcrosslist['goldcrossindex'] = df.loc[df['MA_Cross'] == 1, 'Unnamed: 0'] - beginindex goldcrosslist['goldcrossprice'] = df.loc[df['MA_Cross'] == 1, 'close'] # 取出死叉点 deathcrosslist = pd.DataFrame({'deathcrosstime': df.loc[df['MA_Cross'] == -1, 'strtime']}) deathcrosslist['deathcrossutc'] = df.loc[df['MA_Cross'] == -1, 'utc_time'] deathcrosslist['deathcrossindex'] = df.loc[df['MA_Cross'] == -1, 'Unnamed: 0'] - beginindex deathcrosslist['deathcrossprice'] = df.loc[df['MA_Cross'] == -1, 'close'] goldcrosslist = goldcrosslist.reset_index(drop=True) deathcrosslist = deathcrosslist.reset_index(drop=True) # 生成多仓序列(金叉在前,死叉在后) if goldcrosslist.ix[0, 'goldcrossindex'] < deathcrosslist.ix[0, 'deathcrossindex']: longcrosslist = pd.concat([goldcrosslist, deathcrosslist], axis=1) else: # 如果第一个死叉的序号在金叉前,则要将死叉往上移1格 longcrosslist = pd.concat([goldcrosslist, deathcrosslist.shift(-1).fillna(0)], axis=1) longcrosslist = longcrosslist.set_index(pd.Index(longcrosslist['goldcrossindex']), drop=True) # 生成空仓序列(死叉在前,金叉在后) if deathcrosslist.ix[0, 'deathcrossindex'] < goldcrosslist.ix[0, 'goldcrossindex']: shortcrosslist = pd.concat([deathcrosslist, goldcrosslist], axis=1) else: # 如果第一个金叉的序号在死叉前,则要将金叉往上移1格 shortcrosslist = pd.concat([deathcrosslist, goldcrosslist.shift(-1).fillna(0)], axis=1) shortcrosslist = shortcrosslist.set_index(pd.Index(shortcrosslist['deathcrossindex']), drop=True) # 取出开多序号和开空序号 openlongindex = df.loc[(df['MA_Cross'] == 1) & (df['KDJ_True'] == 1) & (df['DMI_True'] == 1)].index openshortindex = df.loc[(df['MA_Cross'] == -1) & (df['KDJ_True'] == -1) & (df['DMI_True'] == -1)].index # 从多仓序列中取出开多序号的内容,即为开多操作 longopr = longcrosslist.loc[openlongindex] longopr['tradetype'] = 1 longopr.rename(columns={'goldcrosstime': 'opentime', 'goldcrossutc': 'openutc', 'goldcrossindex': 'openindex', 'goldcrossprice': 'openprice', 'deathcrosstime': 'closetime', 'deathcrossutc': 'closeutc', 'deathcrossindex': 'closeindex', 'deathcrossprice': 'closeprice'}, inplace=True) # 从空仓序列中取出开空序号的内容,即为开空操作 shortopr = shortcrosslist.loc[openshortindex] shortopr['tradetype'] = -1 shortopr.rename(columns={'deathcrosstime': 'opentime', 'deathcrossutc': 'openutc', 'deathcrossindex': 'openindex', 'deathcrossprice': 'openprice', 'goldcrosstime': 'closetime', 'goldcrossutc': 'closeutc', 'goldcrossindex': 'closeindex', 'goldcrossprice': 'closeprice'}, inplace=True) # 结果分析 result = pd.concat([longopr, shortopr]) result = result.sort_index() result = result.reset_index(drop=True) result.drop(result.shape[0] - 1, inplace=True) # 去掉跨合约的操作 # 使用单合约,不用再去掉跨合约 # result = removeContractSwap(result, contractswaplist) slip = symbolinfo.getSlip() result['ret'] = ((result['closeprice'] - result['openprice']) * result['tradetype']) - slip result['ret_r'] = result['ret'] / result['openprice'] results = {} ''' # 使用单合约,策略核心内不再计算结果 if calcResult: result['commission_fee'], result['per earn'], result['own cash'], result['hands'] = RS.calcResult(result, symbolinfo, initialCash, positionRatio) endcash = result['own cash'].iloc[-1] Annual = RS.annual_return(result) Sharpe = RS.sharpe_ratio(result) DrawBack = RS.max_drawback(result)[0] SR = RS.success_rate(result) max_single_loss_rate = abs(result['ret_r'].min()) results = { 'Setname':setname, 'opentimes': result.shape[0], 'end_cash': endcash, 'SR': SR, 'Annual': Annual, 'Sharpe': Sharpe, 'DrawBack': DrawBack, 'max_single_loss_rate': max_single_loss_rate } closeopr = result.loc[:, 'closetime':'tradetype'] return result, df, closeopr, results ''' return result
def LvyiNoKDJWin(symbolinfo, setname, rawdata, paraset, contractswaplist, calcResult=True): print setname DMI_N = paraset['DMI_N'] DMI_M = paraset['DMI_M'] MA_Short = paraset['MA_Short'] MA_Long = paraset['MA_Long'] beginindex = rawdata.ix[0, 'Unnamed: 0'] df_DMI = DMI.DMI(rawdata, N=DMI_N, M=DMI_M) df_MA = MA.MA(rawdata['close'], MA_Short, MA_Long) df_MA.drop('close', axis=1, inplace=True) df = pd.concat([rawdata, df_DMI, df_MA], axis=1) # 找出买卖点: # 1.先找出MA金叉的买卖点 # 2.找到结合判决条件的买点 # 3.从MA买点中滤出真实买卖点 # 取出金叉点 goldcrosslist = pd.DataFrame( {'goldcrosstime': df.loc[df['MA_Cross'] == 1, 'strtime']}) goldcrosslist['goldcrossutc'] = df.loc[df['MA_Cross'] == 1, 'utc_time'] goldcrosslist['goldcrossindex'] = df.loc[df['MA_Cross'] == 1, 'Unnamed: 0'] - beginindex goldcrosslist['goldcrossprice'] = df.loc[df['MA_Cross'] == 1, 'close'] # 取出死叉点 deathcrosslist = pd.DataFrame( {'deathcrosstime': df.loc[df['MA_Cross'] == -1, 'strtime']}) deathcrosslist['deathcrossutc'] = df.loc[df['MA_Cross'] == -1, 'utc_time'] deathcrosslist['deathcrossindex'] = df.loc[df['MA_Cross'] == -1, 'Unnamed: 0'] - beginindex deathcrosslist['deathcrossprice'] = df.loc[df['MA_Cross'] == -1, 'close'] goldcrosslist = goldcrosslist.reset_index(drop=True) deathcrosslist = deathcrosslist.reset_index(drop=True) # 生成多仓序列(金叉在前,死叉在后) if goldcrosslist.ix[0, 'goldcrossindex'] < deathcrosslist.ix[ 0, 'deathcrossindex']: longcrosslist = pd.concat([goldcrosslist, deathcrosslist], axis=1) else: # 如果第一个死叉的序号在金叉前,则要将死叉往上移1格 longcrosslist = pd.concat( [goldcrosslist, deathcrosslist.shift(-1).fillna(0)], axis=1) longcrosslist = longcrosslist.set_index(pd.Index( longcrosslist['goldcrossindex']), drop=True) # 生成空仓序列(死叉在前,金叉在后) if deathcrosslist.ix[0, 'deathcrossindex'] < goldcrosslist.ix[ 0, 'goldcrossindex']: shortcrosslist = pd.concat([deathcrosslist, goldcrosslist], axis=1) else: # 如果第一个金叉的序号在死叉前,则要将金叉往上移1格 shortcrosslist = pd.concat( [deathcrosslist, goldcrosslist.shift(-1).fillna(0)], axis=1) shortcrosslist = shortcrosslist.set_index(pd.Index( shortcrosslist['deathcrossindex']), drop=True) #取出开多序号和开空序号 openlongindex = df.loc[(df['MA_Cross'] == 1) & (df['DMI_True'] == 1)].index openshortindex = df.loc[(df['MA_Cross'] == -1) & (df['DMI_True'] == -1)].index # 从多仓序列中取出开多序号的内容,即为开多操作 longopr = longcrosslist.loc[openlongindex] longopr['tradetype'] = 1 longopr.rename(columns={ 'goldcrosstime': 'opentime', 'goldcrossutc': 'openutc', 'goldcrossindex': 'openindex', 'goldcrossprice': 'openprice', 'deathcrosstime': 'closetime', 'deathcrossutc': 'closeutc', 'deathcrossindex': 'closeindex', 'deathcrossprice': 'closeprice' }, inplace=True) # 从空仓序列中取出开空序号的内容,即为开空操作 shortopr = shortcrosslist.loc[openshortindex] shortopr['tradetype'] = -1 shortopr.rename(columns={ 'deathcrosstime': 'opentime', 'deathcrossutc': 'openutc', 'deathcrossindex': 'openindex', 'deathcrossprice': 'openprice', 'goldcrosstime': 'closetime', 'goldcrossutc': 'closeutc', 'goldcrossindex': 'closeindex', 'goldcrossprice': 'closeprice' }, inplace=True) # 结果分析 result = pd.concat([longopr, shortopr]) result = result.sort_index() result = result.reset_index(drop=True) result.drop(result.shape[0] - 1, inplace=True) # 去掉跨合约的操作 result = removeContractSwap(result, contractswaplist) initial_cash = 20000 margin_rate = 0.2 slip = symbolinfo.getSlip() multiplier = symbolinfo.getMultiplier() poundgeType, poundgeFee, poundgeRate = symbolinfo.getPoundage() result['ret'] = ((result['closeprice'] - result['openprice']) * result['tradetype']) - slip result['ret_r'] = result['ret'] / result['openprice'] results = {} if calcResult: firsttradecash = initial_cash / margin_rate result['commission_fee'] = 0 if poundgeType == symbolinfo.POUNDGE_TYPE_RATE: result.ix[0, 'commission_fee'] = firsttradecash * poundgeRate * 2 else: result.ix[0, 'commission_fee'] = firsttradecash / ( multiplier * result.ix[0, 'openprice']) * poundgeFee * 2 result['per earn'] = 0 # 单笔盈亏 result['own cash'] = 0 # 自有资金线 result['trade money'] = 0 # 杠杆后的可交易资金线 result.ix[0, 'per earn'] = firsttradecash * result.ix[0, 'ret_r'] result.ix[0, 'own cash'] = initial_cash + result.ix[ 0, 'per earn'] - result.ix[0, 'commission_fee'] result.ix[0, 'trade money'] = result.ix[0, 'own cash'] / margin_rate oprtimes = result.shape[0] for i in np.arange(1, oprtimes): # 根据手续费类型计算手续费 if poundgeType == symbolinfo.POUNDGE_TYPE_RATE: commission = result.ix[i - 1, 'trade money'] * poundgeRate * 2 else: commission = result.ix[i - 1, 'trade money'] / ( multiplier * result.ix[i, 'openprice']) * poundgeFee * 2 perearn = result.ix[i - 1, 'trade money'] * result.ix[i, 'ret_r'] owncash = result.ix[i - 1, 'own cash'] + perearn - commission result.ix[i, 'own cash'] = owncash result.ix[i, 'commission_fee'] = commission result.ix[i, 'per earn'] = perearn result.ix[i, 'trade money'] = owncash / margin_rate endcash = result.ix[oprtimes - 1, 'own cash'] Annual = RS.annual_return(result) Sharpe = RS.sharpe_ratio(result) DrawBack = RS.max_drawback(result)[0] SR = RS.success_rate(result) max_single_loss_rate = abs(result['ret_r'].min()) results = { 'Setname': setname, 'MA_Short': MA_Short, 'MA_Long': MA_Long, 'DMI_N': DMI_N, 'opentimes': oprtimes, 'end_cash': endcash, 'SR': SR, 'Annual': Annual, 'Sharpe': Sharpe, 'DrawBack': DrawBack, 'max_single_loss_rate': max_single_loss_rate } closeopr = result.loc[:, 'closetime':'tradetype'] return result, df, closeopr, results
# How to use numpy with 'None' value in Python? import MA a = MA.array([1, 2, None], mask = [0, 0, 1]) print "average =", MA.average(a)
def rgrd(self, dataIn=None): """ -------------------------------------------------------------------------------------------------------- ROUTINE: rgrd PURPOSE: To perform one of the following two possible computations in 3-space: To interpolate random data in 3-space using a modified Shepard's algorithm To find the nearest neighbor to a user specified point USAGE: To interpolate use dataOut = r.rgrd(dataIn) where r -- an instance of Shgrid dataIn -- input data dataOut -- output data To locate the nearest point use np = r.rgrd(numberPoints) where r -- an instance of Shgrid np -- the array with numberPoints indices into the input grid idenifying the nearest points --------------------------------------------------------------------------------------------------------""" if self.callGridded == 'yes': # Interpolation in 3-space if usefilled == 'yes': dataIn = MA.filled(dataIn) if debug == 1: print 'calling shgrid' iwk = numpy.zeros((2 * self.ni, ), 'i') rwk = numpy.zeros((11 * self.ni + 6, ), 'f') dataOut, ier = shgridmodule.shgrid(self.xi, self.yi, self.zi, dataIn, self.xo, self.yo, self.zo, iwk, rwk) dataOut = numpy.transpose(dataOut) if debug == 1: print '***************** returning from shgrid with ier = ', ier if ier != 0: msg = 'Error in return from shgrid call with -- ' + Shgrid.errorTable( self)[ier] raise ValueError, msg # is a reverse the order in the returned arrays necessary? if (self.xreverse == 'yes') or (self.yreverse == 'yes') or (self.zreverse == 'yes'): needReverse = 'yes' else: needReverse = 'no' if needReverse == 'yes': dataOut = Shgrid.reverseData(self, dataOut) return dataOut #--------------------------------------------------------------------------------- else: # Computation of nearest neighbors if debug == 1: print 'calling shgetnp' numberPoints = dataIn np = numpy.zeros((numberPoints, ), numpy.int32) n = self.ni iwk = numpy.zeros((2 * n, ), numpy.int32) rwk = numpy.zeros((11 * n + 6, ), numpy.float32) iflag = 0 np[0], ier = shgridmodule.shgetnp(self.xo, self.yo, self.zo, self.xi, self.yi, self.zi, iflag, iwk, rwk) if debug == 1: print '***************** returning from shgrid with ier = ', ier if ier != 0: msg = 'Error in return from shgetnp call with -- ' + Shgrid.errorTable( self)[ier] raise ValueError, msg iflag = 1 for i in range(1, numberPoints): np[i], ier = shgridmodule.shgetnp(self.xo, self.yo, self.zo, self.xi, self.yi, self.zi, iflag, iwk, rwk) if debug == 1: print '***************** returning from shgrid with ier = ', ier if ier != 0: msg = 'Error in return from shgetnp call with -- ' + Shgrid.errorTable( self)[ier] raise ValueError, msg return np
def makeFigure(self): """ :todo: Uses pylab so may not be threadsafe. """ lon = self.grid.lon0 + N.arrayrange(self.grid.nlon) * self.grid.dlon lat = self.grid.lat0 + N.arrayrange(self.grid.nlat) * self.grid.dlat ll_lon, ll_lat = lon[0], lat[0] ur_lon, ur_lat = lon[-1], lat[-1] # Account for variable lat/lon axis ordering #!TODO: Could we move this (along with the equivilent in render_imp.py into grid.py? if self.grid.ilat < self.grid.ilon: latLonOrdering = True else: latLonOrdering = False if latLonOrdering: var = self.grid.value else: var = MA.transpose(self.grid.value) fig = p.figure() map = Basemap(projection='cyl', llcrnrlon=ll_lon, llcrnrlat=ll_lat, urcrnrlon=ur_lon, urcrnrlat=ur_lat, resolution='l') ## if self.grid.units: ## p.title("%s\n(%s)" % (self.grid.long_name, self.grid.units)) ## else: ## p.title(self.grid.long_name) p.title(self.grid.long_name) if self.type == 'colour': # transform_scalar doesn't support masked arrays so we must fill then replace the mask. var_dat = map.transform_scalar(var.filled(1.0e20), lon, lat, len(lon), len(lat)) var = MA.masked_values(var_dat, 1.0e20) map.imshow(var, cmap=self.cmap, vmin=self.vmin, vmax=self.vmax, interpolation='nearest') cbar = p.colorbar(orientation='horizontal', format='%.2g') if self.grid.units: cbar.ax.set_xlabel(self.grid.units) else: x, y = map(*p.meshgrid(lon, lat)) c = map.contour(x, y, var, 12, colors='black') c.clabel(fontsize=8) map.fillcontinents(color='#e0e0e0') map.drawcoastlines(color='gray') map.drawmeridians(p.arange(-180, 180, 30), labels=[1, 0, 0, 1], color='gray') map.drawparallels(p.arange(-90, 90, 15), labels=[1, 0, 0, 1], color='gray') # Wrap the caption caption = word_wrap(self.caption, 80) fig.text(0.1, 0.08, caption, fontsize=10, horizontalalignment='left', verticalalignment='top', transform=fig.transFigure) return fig