def dsetNS(dsetN, dsetS, tasklist = [0,1,3], icovlist=[None,None,None]):
x = dset(fbase=None,wdir=None,tasklist = []) ## do nothing.
x.fbase = dsetN.fbase
x.wdir = dsetN.wdir
if 0 in tasklist:
cut=0.533655
## remove the hard-coding here soon!
try:
indx = np.where(np.array(tasklist) == 0)[0]
icovfname = icovlist[indx]
except:
icovfname = None
tmp = dsetN.xiell + dsetS.xiell
x.xiell = xiell.xiell(icovfname=icovfname, sxilist = [tmp.svec, tmp.xi])
if 1 in tasklist:
try:
indx = np.where(np.array(tasklist) == 1)[0]
icovfname = icovlist[indx]
except:
icovfname = None
tmp = dsetN.wp + dsetS.wp
x.wp = wp.wp(icovfname=icovfname,rpwplist=[tmp.rsig, tmp.wp])
if 3 in tasklist:
try:
indx = np.where(np.array(tasklist) == 3)[0]
icovfname = icovlist[indx]
except:
icovfname = None
x.nbar2d = dsetN.nbar2d
x.nbar3d = dsetN.nbar3d
tmp = dsetN.wpcross + dsetS.wpcross
x.wpcross = wp.wp(icovfname=icovfname,rpwplist=[tmp.rsig, tmp.wp])
return x
def dsetNS(dsetN, dsetS, tasklist=[0, 1, 3], icovlist=[None, None, None]):
x = dset(fbase=None, wdir=None, tasklist=[]) ## do nothing.
x.fbase = dsetN.fbase
x.wdir = dsetN.wdir
if 0 in tasklist:
cut = 0.533655
## remove the hard-coding here soon!
try:
indx = np.where(np.array(tasklist) == 0)[0]
icovfname = icovlist[indx]
except:
icovfname = None
tmp = dsetN.xiell + dsetS.xiell
x.xiell = xiell.xiell(icovfname=icovfname, sxilist=[tmp.svec, tmp.xi])
if 1 in tasklist:
try:
indx = np.where(np.array(tasklist) == 1)[0]
icovfname = icovlist[indx]
except:
icovfname = None
tmp = dsetN.wp + dsetS.wp
x.wp = wp.wp(icovfname=icovfname, rpwplist=[tmp.rsig, tmp.wp])
if 3 in tasklist:
try:
indx = np.where(np.array(tasklist) == 3)[0]
icovfname = icovlist[indx]
except:
icovfname = None
x.nbar2d = dsetN.nbar2d
x.nbar3d = dsetN.nbar3d
tmp = dsetN.wpcross + dsetS.wpcross
x.wpcross = wp.wp(icovfname=icovfname, rpwplist=[tmp.rsig, tmp.wp])
return x
def wpcorrect(wpNNin, wpangin, splitwp, wpstart, wpend):
wpcorrwp = copy.deepcopy(wpangin.wp[wpstart:wpend + 1])
rsigin = copy.deepcopy(wpangin.rsig[wpstart:wpend + 1])
wpcorrwp[splitwp - wpstart:wpend + 1 - wpstart] = wpNNin.wp[splitwp:wpend +
1]
wpcorr = wp.wp(rpwplist=[rsigin, wpcorrwp])
return wpcorr
def __init__(self,
xi2dfname=None,
wpfname=None,
wpicovfname=None,
xiellfname=None,
xiellicovfname=None,
xidatfname=None):
"""
Uses file inputs to read in various correlation function statistics (xi2d, xiell, wp) into a xi object.
The main purpose of this class is to package a single model/measurement together and make a fancy plot.
"""
if xidatfname is not None:
ifp = open(xidatfname, 'r')
print 'reading info from', xidatfname
for line in ifp:
if xi2dfname is None:
if (re.match('xi2d:', line)):
xi2dfname = line.split(':')[1].strip(' \n')
if wpfname is None:
if (re.match('wp:', line)):
x = line.split(':')[1].split(',')
assert len(x) == 1 or len(x) == 2
wpfname = x[0].strip(' \n')
if (len(x) == 2 and wpicovfname is None):
wpicovfname = x[1].strip(' \n')
if xiellfname is None:
if (re.match('xiell:', line)):
x = line.split(':')[1].split(',')
assert len(x) == 1 or len(x) == 2
xiellfname = x[0].strip(' \n')
if (len(x) == 2 and xiellicovfname is None):
xiellicovfname = x[1].strip(' \n')
self.xi2dfname = xi2dfname
self.wpfname = wpfname
self.xiellfname = xiellfname
if (xi2dfname is not None):
self.xi2d = xi2d.xi2d(xi2dfname)
else:
self.xi2d = None
if (wpfname is not None):
self.wp = wp.wp(wpfname, icovfname=wpicovfname)
else:
self.wp = None
if (xiellfname is not None):
self.xiell = xiell.xiell(xiellfname, icovfname=xiellicovfname)
else:
self.xiell = None
def __init__(self,xi2dfname=None,wpfname=None,wpicovfname=None,xiellfname=None,xiellicovfname=None,xidatfname=None):
"""
Uses file inputs to read in various correlation function statistics (xi2d, xiell, wp) into a xi object.
The main purpose of this class is to package a single model/measurement together and make a fancy plot.
"""
if xidatfname is not None:
ifp = open(xidatfname,'r')
print 'reading info from',xidatfname
for line in ifp:
if xi2dfname is None:
if(re.match('xi2d:',line)):
xi2dfname = line.split(':')[1].strip(' \n')
if wpfname is None:
if(re.match('wp:',line)):
x = line.split(':')[1].split(',')
assert len(x) == 1 or len(x) == 2
wpfname = x[0].strip(' \n')
if(len(x) == 2 and wpicovfname is None):
wpicovfname = x[1].strip(' \n')
if xiellfname is None:
if(re.match('xiell:',line)):
x = line.split(':')[1].split(',')
assert len(x) == 1 or len(x) == 2
xiellfname = x[0].strip(' \n')
if(len(x) == 2 and xiellicovfname is None):
xiellicovfname = x[1].strip(' \n')
self.xi2dfname = xi2dfname
self.wpfname = wpfname
self.xiellfname = xiellfname
if(xi2dfname is not None):
self.xi2d = xi2d.xi2d(xi2dfname)
else: self.xi2d = None
if(wpfname is not None):
self.wp = wp.wp(wpfname,icovfname=wpicovfname)
else: self.wp = None
if(xiellfname is not None):
self.xiell = xiell.xiell(xiellfname,icovfname=xiellicovfname)
else: self.xiell = None
def psh_help():
wp("""psh: shell implementation in Python.
Supports all basic shell commands.""")
#!/usr/bin/env python3
import random, sys, time, os, re
from wp import printt as wp
#from echo import echo as echo
wp("welcome to psh...")
wp("made by Sakurai07")
wp("run help.py for psh help")
"""psh: a simple shell written in Python"""
import os
import subprocess
def execute_command(command):
"""execute commands and handle piping"""
try:
if "|" in command:
# save for restoring later on
s_in, s_out = (0, 0)
s_in = os.dup(0)
s_out = os.dup(1)
# first command takes commandut from stdin
fdin = os.dup(s_in)
# iterate over all the commands that are piped
for cmd in command.split("|"):
# fdin will be stdin if it's the first iteration
# and the readable end of the pipe if not.
os.dup2(fdin, 0)
def wpcorrect(wpNNin, wpangin, splitwp, wpstart, wpend):
wpcorrwp = copy.deepcopy(wpangin.wp[wpstart : wpend + 1])
rsigin = copy.deepcopy(wpangin.rsig[wpstart : wpend + 1])
wpcorrwp[splitwp - wpstart : wpend + 1 - wpstart] = wpNNin.wp[splitwp : wpend + 1]
wpcorr = wp.wp(rpwplist=[rsigin, wpcorrwp])
return wpcorr
def debiasdataandcovwp(
wpNNd, wpangd, wpangdhigh, wpangdlow, wpNNm, wpangm, wp012m, splitwp, wpstart, wpend, covstatfname, fname=None
):
"""
subtract the bias measured from the tiled mocks from the data, return a debiased combination.
print it to a file (fname) to be fed to bethalexie code in long format.
Also take in statistical covariance matrix and add two sources of systematics.
"""
wpcorrdtmp = wpcorrect(wpNNd, wpangd, splitwp, wpstart, wpend)
wpcorrm = wpcorrect(wpNNm, wpangm, splitwp, wpstart, wpend)
wpdebiased = copy.deepcopy(wpcorrdtmp.wp)
mydelta = wp012m.wp[wpstart:] - wpcorrm.wp
print "fractional wp correction:"
print mydelta / wpcorrdtmp.wp
wpdebiased = wpdebiased + mydelta
## now the cov.
## make sure this is the cov for the corrected statistic with same splits.
if 0 == 0:
# try:
cov = np.loadtxt(covstatfname)
assert len(cov[:, 0]) == len(wpdebiased)
splitz = covstatfname.split("splitswp")[1].split("_")
assert len(splitz) >= 2
ilist = []
for ss in splitz[:2]:
ilist.append(int(ss))
assert ilist[0] == splitwp
assert ilist[1] == wpstart
## new jan 2 2014!!! forgot to take into account the unbiasicov fac. derive if from
## product of cov and icov.
## guess icovfname
tmp = covstatfname.split("/")
tmp[-1] = "i" + tmp[-1]
icovstatfname = "/".join(tmp)
icov = np.loadtxt(icovstatfname)
unbiasicovfac = (ximisc.getmatrixdiag(np.matrix(cov) * np.matrix(icov))).mean()
print "using htis unbiasicovfac correction, dividing cov by this", unbiasicovfac
cov = cov / unbiasicovfac
ndatacorr = len(wpcorrdtmp.wp)
diagstat = np.zeros(ndatacorr)
diagtot = np.zeros(ndatacorr)
diagvar = np.zeros(ndatacorr)
for i in range(len(diagstat)):
diagstat[i] = cov[i, i]
## this must agree with wpcorrect assignmeents!
wpangdiffvar = (0.5 * (wpangdhigh.wp - wpangdlow.wp)) ** 2
diagvar[0 : splitwp - wpstart] = wpangdiffvar[wpstart:splitwp]
print "wp ang high/low variance: ", diagvar / diagstat
print "bias correction: ", mydelta
diagvar = diagvar + (mydelta.flatten()) ** 2
print "bias variance contribution: ", (mydelta.flatten()) ** 2 / diagstat
## add it into the covarianace matrix.
for i in range(ndatacorr):
cov[i, i] += diagvar[i]
diagtot[i] = cov[i, i]
print "total sys variance fraction", diagtot / diagstat
## make it a matrix.
cov = np.matrix(cov)
icov = cov.I
fcovout = covstatfname + ".sys"
## print the covariance and icov to new file.
printcov(cov, fcovout)
tmp = fcovout.split("/")
tmp[-1] = "i" + tmp[-1]
ifcovout = "/".join(tmp)
printcov(icov, ifcovout)
wpfinal = wp.wp(rpwplist=[wpcorrdtmp.rsig, wpdebiased], icovfname=ifcovout)
if fname is not None:
wpfinal.printwp(fname)
return wpfinal, cov
else:
# except:
print "cov file name does not match input splits, returning None!"
wpfinal = wp.wp(rpwplist=[wpcorrdtmp.rsig, wpdebiased])
if fname is not None:
wpfinal.printwp(fname)
return wpfinal, None
def debiasdataandcovwp(wpNNd,
wpangd,
wpangdhigh,
wpangdlow,
wpNNm,
wpangm,
wp012m,
splitwp,
wpstart,
wpend,
covstatfname,
fname=None):
"""
subtract the bias measured from the tiled mocks from the data, return a debiased combination.
print it to a file (fname) to be fed to bethalexie code in long format.
Also take in statistical covariance matrix and add two sources of systematics.
"""
wpcorrdtmp = wpcorrect(wpNNd, wpangd, splitwp, wpstart, wpend)
wpcorrm = wpcorrect(wpNNm, wpangm, splitwp, wpstart, wpend)
wpdebiased = copy.deepcopy(wpcorrdtmp.wp)
mydelta = wp012m.wp[wpstart:] - wpcorrm.wp
print 'fractional wp correction:'
print mydelta / wpcorrdtmp.wp
wpdebiased = wpdebiased + mydelta
## now the cov.
## make sure this is the cov for the corrected statistic with same splits.
if (0 == 0):
# try:
cov = np.loadtxt(covstatfname)
assert len(cov[:, 0]) == len(wpdebiased)
splitz = covstatfname.split('splitswp')[1].split('_')
assert len(splitz) >= 2
ilist = []
for ss in splitz[:2]:
ilist.append(int(ss))
assert ilist[0] == splitwp
assert ilist[1] == wpstart
## new jan 2 2014!!! forgot to take into account the unbiasicov fac. derive if from
## product of cov and icov.
## guess icovfname
tmp = covstatfname.split('/')
tmp[-1] = 'i' + tmp[-1]
icovstatfname = '/'.join(tmp)
icov = np.loadtxt(icovstatfname)
unbiasicovfac = (ximisc.getmatrixdiag(
np.matrix(cov) * np.matrix(icov))).mean()
print 'using htis unbiasicovfac correction, dividing cov by this', unbiasicovfac
cov = cov / unbiasicovfac
ndatacorr = len(wpcorrdtmp.wp)
diagstat = np.zeros(ndatacorr)
diagtot = np.zeros(ndatacorr)
diagvar = np.zeros(ndatacorr)
for i in range(len(diagstat)):
diagstat[i] = cov[i, i]
## this must agree with wpcorrect assignmeents!
wpangdiffvar = (0.5 * (wpangdhigh.wp - wpangdlow.wp))**2
diagvar[0:splitwp - wpstart] = wpangdiffvar[wpstart:splitwp]
print 'wp ang high/low variance: ', diagvar / diagstat
print 'bias correction: ', mydelta
diagvar = diagvar + (mydelta.flatten())**2
print 'bias variance contribution: ', (mydelta.flatten())**2 / diagstat
## add it into the covarianace matrix.
for i in range(ndatacorr):
cov[i, i] += diagvar[i]
diagtot[i] = cov[i, i]
print 'total sys variance fraction', diagtot / diagstat
## make it a matrix.
cov = np.matrix(cov)
icov = cov.I
fcovout = covstatfname + '.sys'
## print the covariance and icov to new file.
printcov(cov, fcovout)
tmp = fcovout.split('/')
tmp[-1] = 'i' + tmp[-1]
ifcovout = '/'.join(tmp)
printcov(icov, ifcovout)
wpfinal = wp.wp(rpwplist=[wpcorrdtmp.rsig, wpdebiased],
icovfname=ifcovout)
if fname is not None:
wpfinal.printwp(fname)
return wpfinal, cov
else:
# except:
print 'cov file name does not match input splits, returning None!'
wpfinal = wp.wp(rpwplist=[wpcorrdtmp.rsig, wpdebiased])
if fname is not None:
wpfinal.printwp(fname)
return wpfinal, None
def post_woo_products(self):
'''Post WooCommerce product'''
try:
# Auth
auth = self.get_woo_request()
# Upload image to media
image = wp(self.book["image"])
data = {
"name":
self.book["name"],
"description":
self.book["description"],
"sku":
self.book["isbn"],
"categories": [],
"tags": [],
"attributes": [
{
"id": 1,
"name": "Tytuł", # cspell: disable-line
"position": 1,
"visible": True,
"variation": True,
"options": [self.book["title"]]
},
{
"id": 2,
"name": "Autor", # cspell: disable-line
"position": 2,
"visible": True,
"variation": True,
"options": [self.book["authors"]]
},
{
"id": 3,
"name": "Wydawnictwo", # cspell: disable-line
"position": 3,
"visible": True,
"variation": True,
"options": [self.book["publisher"]]
},
{
"id": 4,
"name": "Rok wydania", # cspell: disable-line
"position": 4,
"visible": True,
"variation": True,
"options": [self.book["publish_date"]]
},
{
"id": 5,
"name": "Okładka", # cspell: disable-line
"position": 5,
"visible": True,
"variation": True,
"options": [self.book["binding"]]
},
{
"id": 6,
"name": "ISBN",
"position": 6,
"visible": True,
"variation": True,
"options": [self.book["isbn"]]
}
]
}
# Tags
try:
if self.book["tags"]:
tags = self.validate_tags()
for tag in tags:
data["tags"].append({'id': tag})
except Exception as error: # pylint: disable=broad-except
logger.info(error)
# Image
try:
if image:
data["images"] = [{"src": image}]
except Exception as error: # pylint: disable=broad-except
logger.info(error)
# Price
try:
if self.book["price"]:
data["regular_price"] = self.book["price"]
except Exception as error: # pylint: disable=broad-except
logger.info(error)
# Sale Price
try:
if self.book["sale_price"]:
data["sale_price"] = self.book["sale_price"]
except Exception as error: # pylint: disable=broad-except
logger.info(error)
# Amount
try:
if self.book["amount"]:
data["manage_stock"] = True
data["stock_quantity"] = self.book["amount"]
except Exception as error: # pylint: disable=broad-except
logger.info(error)
# Get category ID
try:
categories = self.validate_category()
for category in categories:
data["categories"].append({'id': category})
except Exception as error: # pylint: disable=broad-except
logger.info(error)
# Send request
response = auth.post("products", data).json()
# Send none if status code found in error codes
if "data" in response:
if response.get("data", {}).get("status") in self.error_codes:
self.error_catch.append(inspect.getouterframes(inspect.currentframe())[0].function) # pylint: disable=line-too-long
return None
# Format output
try:
output = {
'id': response["id"],
'name': response["name"],
'link': response["permalink"],
'source': False
}
except Exception as error: # pylint: disable=broad-except
logger.info(error)
if response["data"]["status"] == 400:
try:
mysql_request = MySQL(isbn=self.book["isbn"])
request = mysql_request.db_mysql()
except Exception as error: # pylint: disable=broad-except
logger.info(error)
if request:
product = self.get_woo_product(request)
if product["stock_quantity"]:
data["stock_quantity"] = int(data["stock_quantity"]) + product["stock_quantity"] # pylint: disable=line-too-long
try:
response = self.update_woo_products(
product["id"], data)
output = {
'id': response["id"],
'name': response["name"],
'link': response["permalink"],
'source': True
}
return output
except Exception as error: # pylint: disable=broad-except
logger.info(error)
else:
return None
except Exception as error: # pylint: disable=broad-except
logger.info(error)
return output
def __init__(self,
xiellin=None,
wpin=None,
xiellwpfname=None,
icovfname=None):
if xiellin is not None:
self.xiell = xiellin
if wpin is not None:
self.wp = wpin
if xiellwpfname is not None:
self.xiellwpfname = xiellwpfname
if xiellin is not None and wpin is not None:
self.ntot = self.wp.nrsig + self.xiell.ndata
self.xiwp = np.concatenate((self.xiell.xilong, self.wp.wp))
if len(self.xiwp) != self.ntot:
print 'vector length mismatch!'
self = None
else:
if xiellwpfname is None:
self = None
return
try:
rx, self.xiwp = np.loadtxt(xiellwpfname,
usecols=[0, 1],
unpack=True)
self.ntot = len(self.xiwp)
except:
self = None
if self is not None:
try:
## find xiell/wp split, create
xx = np.where(rx[:-1] > rx[1:])[0]
## i *think* this should work, double check if you choose some wacko binning!
xwp = xx[-1] + 1
nell = len(xx)
svec = rx[:xwp].reshape(nell, len(rx[:xwp]) / nell)
xivec = self.xiwp[:xwp].reshape(nell, len(rx[:xwp]) / nell)
self.xiell = xiell.xiell(sxilist=[svec, xivec])
self.wp = wp.wp(rpwplist=[rx[xwp:], self.xiwp[xwp:]])
except: # oh well.
pass
## tmp.
# self.xiell = xiell.xiell(sxilist=[rx[:xwp],self.xiwp[:xwp]])
# self.wp = wp.wp(rpwplist=[rx[xwp:], self.xiwp[xwp:]])
if self is not None:
if icovfname is None:
self.DorT = 1
else:
self.DorT = 0
icov = np.matrix(np.loadtxt(icovfname))
self.icov = icov
try:
cov = icov.I
diagerr = np.array(
[np.sqrt(cov[i, i]) for i in range(self.ntot)])
self.diagerr = diagerr
except:
diagerr = np.zeros(self.ntot) + 1.
self.diagerr = diagerr
if not len(icov[:, 0]) == self.ntot:
self = None
def __init__(self,xiellin=None,wpin=None,xiellwpfname=None,icovfname=None):
if xiellin is not None:
self.xiell = xiellin
if wpin is not None:
self.wp = wpin
if xiellwpfname is not None:
self.xiellwpfname = xiellwpfname
if xiellin is not None and wpin is not None:
self.ntot = self.wp.nrsig + self.xiell.ndata
self.xiwp = np.concatenate((self.xiell.xilong, self.wp.wp))
if len(self.xiwp) != self.ntot:
print 'vector length mismatch!'
self = None
else:
if xiellwpfname is None:
self = None
return
try:
rx, self.xiwp = np.loadtxt(xiellwpfname,usecols=[0,1],unpack=True)
self.ntot = len(self.xiwp)
except:
self = None
if self is not None:
try:
## find xiell/wp split, create
xx = np.where(rx[:-1] > rx[1:])[0]
## i *think* this should work, double check if you choose some wacko binning!
xwp = xx[-1]+1
nell = len(xx)
svec = rx[:xwp].reshape(nell,len(rx[:xwp])/nell)
xivec = self.xiwp[:xwp].reshape(nell,len(rx[:xwp])/nell)
self.xiell = xiell.xiell(sxilist=[svec,xivec])
self.wp = wp.wp(rpwplist=[rx[xwp:], self.xiwp[xwp:]])
except: # oh well.
pass
## tmp.
# self.xiell = xiell.xiell(sxilist=[rx[:xwp],self.xiwp[:xwp]])
# self.wp = wp.wp(rpwplist=[rx[xwp:], self.xiwp[xwp:]])
if self is not None:
if icovfname is None:
self.DorT = 1
else:
self.DorT = 0
icov = np.matrix(np.loadtxt(icovfname))
self.icov = icov
try:
cov = icov.I
diagerr = np.array([np.sqrt(cov[i,i]) for i in range(self.ntot)])
self.diagerr = diagerr
except:
diagerr = np.zeros(self.ntot) + 1.
self.diagerr = diagerr
if not len(icov[:,0]) == self.ntot:
self = None