def plotMapFrame(time,
UA_sites,
BLAT,
BLNG,
DRV_COL,
GC_FRA,
lngs,
lats,
EXP_VID,
offset=2.5,
amplitude=2,
alpha=.35,
marker=(6, 0),
DPI=250,
edgecolor='#ffffff'):
print('* Exporting {}'.format(str(time).zfill(4)), end='\r')
# Create map --------------------------------------------------------------
(fig, ax) = plt.subplots(figsize=(10, 10))
(fig, ax, mapR) = plotMap(fig, ax, UA_sites, BLAT, BLNG, ptColor='#6347ff')
# Pops --------------------------------------------------------------------
(fig, ax, mapR) = plotGenePopsOnMap(fig,
ax,
mapR,
lngs,
lats,
DRV_COL,
GC_FRA,
time,
edgecolor=edgecolor,
marker=marker,
offset=offset,
amplitude=amplitude,
alpha=alpha)
ax.text(0.75,
0.1,
str(time).zfill(4),
horizontalalignment='center',
verticalalignment='center',
transform=ax.transAxes,
fontsize=30)
fun.quickSaveFig('{}/{}.png'.format(EXP_VID,
str(time).zfill(4)),
fig,
dpi=DPI)
plt.close(fig)
(header, xpidIx) = (('i_rer', 'i_ren', 'i_rsg', 'i_fic', 'i_gsv', 'i_grp'),
(1, 2, 3, 4, 5, 7))
outLabels = ('TTI', 'TTO', 'WOP', 'RAP', 'MNX')
(PT_ROT, PT_IMG, PT_DTA, PT_PRE, PT_OUT,
PT_MTR) = aux.selectPath(USR, LND, REL)
###############################################################################
# Setup schemes
###############################################################################
tS = datetime.now()
aux.printExperimentHead(PT_ROT, PT_IMG, PT_OUT, tS, 'PostProcess ' + AOI)
# Setting up tha names for output files ---------------------------------------
pth = PT_MTR + AOI + '_{}_' + QNT + '_qnt.csv'
DFOPths = [pth.format(z) for z in outLabels]
# Setup experiments IDs -------------------------------------------------------
uids = fun.getExperimentsIDSets(PT_OUT, skip=-1)
(rer, ren, rsg, fic, gsv, aoi, grp) = uids[1:]
(xpDict, smryDicts) = ({}, ({}, {}, {}, {}, {}))
# Get experiment files --------------------------------------------------------
ptrn = aux.XP_NPAT.format('*', '*', '*', '*', '*', AOI, '*', 'rto', 'npy')
fPaths = sorted(glob(PT_OUT + ptrn))
(fNum, digs) = monet.lenAndDigits(fPaths)
qnt = float(int(QNT) / 100)
# Setup dataframes ------------------------------------------------------------
outDFs = monet.initDFsForDA(fPaths, header, thiS, thoS, thwS, tapS)
(ttiDF, ttoDF, wopDF, tapDF, rapDF) = outDFs
###############################################################################
# Iterate through experiments
###############################################################################
fmtStr = '{}+ File: {}/{}'
for (i, fPath) in enumerate(fPaths):
(DRV, MGV) = aux.humanSelector(AOI, DRV, MGV)
(PT_ROT, PT_IMG, PT_DTA, PT_PRE, PT_OUT, PT_MTR) = aux.selectPath(USR, LND, REL)
(drive, land) = (
drv.driveSelector(DRV, AOI, popSize=10*10000),
lnd.landSelector(LND, REL, PT_ROT)
)
(gene, fldr) = (drive.get('gDict'), drive.get('folder'))
# Time and head ---------------------------------------------------------------
tS = datetime.now()
monet.printExperimentHead(PT_ROT, PT_PRE, tS, 'UCIMI Preprocess '+AOI)
###############################################################################
# Load folders
###############################################################################
(expDirsMean, expDirsTrac) = aux.selectVersionPath(MGV, PT_DTA)
(expNum, nodeDigits) = (len(expDirsMean), len(str(len(land)))+1)
outNames = fun.splitExpNames(PT_OUT)
outExpNames = set(outNames)
###############################################################################
# Analyze data
###############################################################################
Parallel(n_jobs=JOB)(
delayed(monet.preProcess)(
exIx, expNum, expDirsMean, expDirsTrac, gene,
analysisOI=AOI, prePath=PT_PRE, nodesAggLst=land,
outExpNames=outExpNames, fNameFmt='{}/{}-{}_', OVW=OVW,
MF=MF, cmpr=FMT, nodeDigits=nodeDigits,
SUM=SUM, AGG=AGG, SPA=SPA, REP=REP, SRP=SRP,
sexFilenameIdentifiers={"male": "M_", "female": "F_"}
) for exIx in range(0, expNum)
)
sns.stripplot(x=cz, jitter=jitter, size=size, color=clrs[2], ax=ax[2], alpha=alpha)
for axt in ax:
axt.set_ylim(-height, height)
axt.set_xlim(0, 10 * 365)
axt.set_yticklabels('')
axt.set_xticklabels('')
axt.set_xticks(np.arange(0, 365 * 10, 365))
axt.set_title('')
axt.set(xlabel=None)
[t.set_visible(False) for t in axt.get_yticklines()]
[t.set_visible(False) for t in axt.get_xticklines()]
axt.spines['top'].set_visible(False)
axt.spines['right'].set_visible(False)
axt.spines['bottom'].set_visible(False)
axt.spines['left'].set_visible(False)
fun.quickSaveFig(PT_IMG + MTR + '_tResponse.png', fig, transparent=False, dpi=500)
###############################################################################
# Density
###############################################################################
plt.rcParams.update({
"figure.facecolor": (1.0, 0.0, 0.0, 0), # red with alpha = 30%
"axes.facecolor": (0.0, 1.0, 0.0, 0), # green with alpha = 50%
"savefig.facecolor": (1.0, 1.0, 1.0, 1), # blue with alpha = 20%
})
fig = plt.figure(figsize=(7, .75))
gs = gridspec.GridSpec(
nrows=3, ncols=1, figure=fig,
width_ratios= [1], height_ratios=[.5, .5, .5],
wspace=0.3, hspace=-.85
)
df = (az, bz, cz)
mL.drawcoastlines(color=COLORS[4], linewidth=10, zorder=0)
ax.tick_params(
axis='both', which='both',
bottom=True, top=False, left=True, right=False,
labelbottom=True, labelleft=True
)
ax.spines["top"].set_visible(False)
ax.spines["right"].set_visible(False)
ax.spines["bottom"].set_visible(False)
ax.spines["left"].set_visible(False)
mH.scatter(
list(pts['lon']), list(pts['lat']), latlon=True,
alpha=.2, marker='.', s=[10 * math.log(i) for i in list(pts['pop'])],
color='#E048B8', zorder=3
)
fun.quickSaveFig(PTH_pts + 'ST.png', fig, dpi=750)
# #############################################################################
# Principe
# #############################################################################
(minLat, minLong) = (1.5, 7.275)
(maxLat, maxLong) = (1.75, 7.5)
fig = plt.figure(figsize=(5, 5))
ax = fig.add_subplot(111, label="1")
mH = Basemap(
projection='merc',
llcrnrlat=minLat, urcrnrlat=maxLat,
llcrnrlon=minLong, urcrnrlon=maxLong,
lat_ts=20, resolution='h', ax=ax
)
mH.drawcoastlines(color=COLORS[0], linewidth=2, zorder=1)
mH.drawcoastlines(color=COLORS[3], linewidth=.25, zorder=2)
for (xpNumC, xpId) in enumerate(idTuples):
xpNumCS = str(xpNumC + 1).zfill(4)
print('* Exporting {}/{}'.format(xpNumCS, xpNumS), end='\r')
#######################################################################
# Filter
#######################################################################
indepFltrs = [list(df[hId[1]] == hId[0]) for hId in zip(xpId, hdFree)]
fullFilter = list(map(all, zip(*indepFltrs)))
dfSrf = df[fullFilter]
#######################################################################
# Plot
#######################################################################
# Prepare the response surface ----------------------------------------
(x, y, z) = (dfSrf[HD_IND[0]], dfSrf[HD_IND[1]], dfSrf[HD_DEP])
(a, b) = ((min(x), max(x)), (min(y), max(y)))
rs = fun.calcResponseSurface(x, y, z, scalers=sclr, mthd=mthd)
(rsG, rsS) = (rs['grid'], rs['surface'])
# Plot the response surface -------------------------------------------
(fig, ax) = plt.subplots(figsize=(8, 7))
# Experiment points, contour lines, response surface
xy = ax.plot(rsG[0], rsG[1], 'k.', ms=3, alpha=.25, marker='.')
cc = ax.contour(rsS[0],
rsS[1],
rsS[2],
levels=lvls,
colors='w',
linewidths=1,
alpha=.5)
cs = ax.contourf(rsS[0],
rsS[1],
rsS[2],
from glob import glob
import STP_aux as aux
import STP_functions as fun
import STP_dataProcess as da
from datetime import datetime
import compress_pickle as pkl
import MoNeT_MGDrivE as monet
(USR, AOI, REL, LND) = (sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4])
# (USR, AOI, REL, LND) = ('dsk', 'HLT', 'gravidFemale', 'PAN')
(DRV, SKP, QNT, OVW) = ('LDR', False, '75', True)
(gIx, hIx) = (1, 0)
(PT_ROT, PT_IMG, PT_DTA, PT_PRE, PT_OUT,
PT_MTR) = aux.selectPath(USR, LND, REL)
uids = fun.getExperimentsIDSets(PT_PRE, skip=-1)
(rer, ren, rsg, fic, gsv, aoi, grp) = uids[1:]
tS = datetime.now()
monet.printExperimentHead(PT_PRE, PT_OUT, tS, 'UCIMI PstFraction ' + AOI)
# #########################################################################
# Base experiments
# These are the experiments without any releases (for fractions)
# #########################################################################
# basePat = aux.XP_NPAT.format('*', '00', '*', '*', '*', AOI, '*', 'sum', 'bz')
basePat = aux.patternForReleases('00', AOI, 'sum')
baseFiles = sorted(glob(PT_PRE + basePat))
# #########################################################################
# Probe experiments
# sum: Analyzed data aggregated into one node
# srp: Garbage data aggregated into one node
# #########################################################################
ax.spines["right"].set_visible(False)
ax.spines["bottom"].set_visible(False)
ax.spines["left"].set_visible(False)
(lon, lat) = (list(pts['lon']), list(pts['lat']))
mH.scatter(lon,
lat,
latlon=True,
alpha=.25,
marker='.',
s=[2],
color='#E048B8',
zorder=3)
for i in range(len(lon)):
(x, y) = mH(lon[i], lat[i])
ax.annotate(i, xy=(x, y), size=.2, ha='center', va='center', color='white')
fun.quickSaveFig(PTH_PTS + 'raw.png', fig, dpi=1000)
mH.scatter([i[0] for i in centroid], [i[1] for i in centroid],
latlon=True,
alpha=.5,
marker='x',
s=[5],
color='#233090',
zorder=3)
for i in range(len(centroid)):
(x, y) = mH(centroid[i][0], centroid[i][1])
ax.annotate(i, xy=(x, y), size=2, ha='center', va='center')
fun.quickSaveFig(PTH_PTS + 'clusters.png', fig, dpi=1000)
# #############################################################################
# Kernel Heatmap
# #############################################################################
kernel = np.genfromtxt(PTH_PTS + kernelName, delimiter=',')
else:
(CLR, CMAPS, YRAN) = (drv.COLHN, drv.COLHM, [0, 100 * 12000 / 2])
STYLE = {
"width": .5,
"alpha": .15,
"dpi": 2 * 300,
"legend": True,
"aspect": .25,
"colors": CLR,
"xRange": [0, 365 * 3],
"yRange": YRAN
}
STYLE['aspect'] = monet.scaleAspect(1, STYLE)
tS = datetime.now()
fun.printExperimentHead(PT_ROT, PT_IMG, PT_PRE, tS, 'Traces')
###############################################################################
# Load preprocessed files lists
###############################################################################
tyTag = ('sum', 'srp')
if FZ:
fLists = list(
zip(*[
fun.getFilteredFiles(PT_PRE + '*_00_*' + AOI + '*' + tp +
'*', PT_PRE + '*' + AOI + '*' + tp + '*')
for tp in tyTag
]))
else:
fLists = list(
zip(*[
sorted(glob(PT_PRE + '*' + AOI + '*' + tp + '*')) for tp in tyTag
(fltrPattern, globPattern) = ('dummy', PT_PRE + '*' + AOI + '*srp*')
if FZ:
fltrPattern = PT_PRE + '*_00_*' + AOI + '*srp*'
repFiles = monet.getFilteredFiles(fltrPattern, globPattern)
repFiles.reverse()
###########################################################################
# Iterate through experiments
###########################################################################
(fNum, digs) = monet.lenAndDigits(repFiles)
fmtStr = '{}+ File: {}/{}'
(i, repFile) = (0, repFiles[0])
for (i, repFile) in enumerate(repFiles):
padi = str(i + 1).zfill(digs)
print(fmtStr.format(monet.CBBL, padi, fNum, monet.CEND), end='\r')
(repDta, xpid) = (pkl.load(repFile),
fun.getXpId(repFile, (1, 2, 3, 4, 5, 7)))
xpRow = [
da.filterDFWithID(i, xpid)
for i in (dfTTI, dfTTO, dfWOP, dfMNX, dfPOE, dfCPT)
]
(tti, tto, wop) = [float(row[THS]) for row in xpRow[:3]]
(mnf, mnd, poe, cpt) = (float(xpRow[3]['min']), float(xpRow[3]['minx']),
float(xpRow[4]['POE']), float(xpRow[5]['CPT']))
# Traces ------------------------------------------------------------------
if LND == 'PAN':
tStable = 0
pop = repDta['landscapes'][0][tStable][-1]
STYLE['yRange'] = (0, pop + pop * .5)
if AOI == 'ECO':
STYLE['yRange'] = (STYLE['yRange'][0], STYLE['yRange'][1] * 2)
STYLE['aspect'] = monet.scaleAspect(1, STYLE)
# Template layout
###############################################################################
template = path.join(PT_IMG, REL + '_template.svg')
with open(template, 'r') as file:
filedata = file.read()
###############################################################################
# Replacements
###############################################################################
PTRN = 'E_{}_{}_{}_{}_{}-{}'
# Find original ids -----------------------------------------------------------
p = re.compile('E_.*png', re.IGNORECASE)
pats = re.findall(p, filedata)
img_org = list({i[:-8] for i in pats})
print(img_org)
# Replacement ids -------------------------------------------------------------
uids = fun.getExperimentsIDSets(path.join(PT_IMG, 'preTraces/'), skip=-1)
uids[2] = uids[2][1:] # Delete the zero releases entry
expSets = set(list(product(*uids[1:-1])))
print(expSets)
###############################################################################
# Iterate
###############################################################################
tS = datetime.now()
monet.printExperimentHead(PT_ROT, PT_IMG, tS, 'UCIMI PrePanels ' + AOI)
(xpNum, digs) = monet.lenAndDigits(expSets)
i = 0
exp = sorted(list(expSets))[-18]
for (i, exp) in enumerate(sorted(list(expSets))):
monet.printProgress(i + 1, xpNum, digs)
# Get exp ids -------------------------------------------------------------
(i_rer, i_ren, i_rsg, i_fic, i_gsv) = exp
from joblib import Parallel, delayed
(USR, AOI, REL, LND) = (sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4])
# (USR, AOI, REL, LND) = ('dsk', 'HLT', 'gravidFemale', 'PAN')
(DRV, FMT, OVW, MF, JOB) = ('LDR', 'bz2', True, (False, True), 6)
(SUM, AGG, SPA, REP, SRP) = (True, False, False, False, True)
###############################################################################
# Setting up paths and style
###############################################################################
(PT_ROT, PT_IMG, PT_DTA, PT_PRE, PT_OUT,
PT_MTR) = aux.selectPath(USR, LND, REL)
(drive, land) = (drv.driveSelector(DRV), lnd.landSelector(LND))
gene = drive.get(AOI).get('gDict')
# Time and head ---------------------------------------------------------------
tS = datetime.now()
fun.printExperimentHead(PT_ROT, PT_IMG, PT_PRE, tS, 'Preprocess ' + AOI)
###############################################################################
# Load folders
###############################################################################
(expDirsMean, expDirsTrac) = fun.getExpPaths(PT_DTA)
(expNum, nodeDigits) = (len(expDirsMean), len(str(len(land))) + 1)
outNames = fun.splitExpNames(PT_OUT)
outExpNames = set(outNames)
###############################################################################
# Analyze data
###############################################################################
Parallel(n_jobs=JOB)(delayed(monet.preProcess)(exIx,
expNum,
expDirsMean,
expDirsTrac,
gene,
# #############################################################################
popsMatch = len(GC_FRA) == len(AGG_lonlats)
# #############################################################################
# Map
# #############################################################################
# Coordinates -----------------------------------------------------------------
(lngs, lats) = (AGG_centroids[:, 0], AGG_centroids[:, 1])
for time in range(GC_FRA[0].shape[0]):
print('* Exporting {}'.format(str(time).zfill(4)), end='\r')
# Create map --------------------------------------------------------------
(fig, ax) = plt.subplots(figsize=(10, 10))
(fig, ax, mapR) = plo.plotMap(
fig, ax, UA_sites, BLAT, BLNG, ptColor='#6347ff'
)
# Pops --------------------------------------------------------------------
(fig, ax, mapR) = plo.plotGenePopsOnMap(
fig, ax, mapR,
lngs, lats, DRV_COL,
GC_FRA, time,
marker=(6, 0), offset=2, amplitude=10
)
ax.text(
0.75, 0.1, str(time).zfill(4),
horizontalalignment='center', verticalalignment='center',
transform=ax.transAxes, fontsize=30
)
fun.quickSaveFig(
'{}/{}.png'.format(EXP_VID, str(time).zfill(4)),
fig, dpi=500
)
plt.close('all')
STYLE['aspect'] = monet.scaleAspect(1, STYLE)
tS = datetime.now()
aux.printExperimentHead(PT_ROT, PT_IMG, PT_OUT, tS, 'PstTraces')
###########################################################################
# Load postprocessed files
###########################################################################
pstPat = PT_MTR + AOI + '_{}_' + QNT + '_qnt.csv'
pstFiles = [pstPat.format(i) for i in ('TTI', 'TTO', 'WOP', 'MNX', 'RAP')]
(dfTTI, dfTTO, dfWOP, dfMNX, _) = [pd.read_csv(i) for i in pstFiles]
###########################################################################
# Load preprocessed files lists
###########################################################################
repFiles = glob(PT_PRE + '*' + AOI + '*' + 'srp' + '*')
if FZ:
repFiles = fun.getFilteredFiles(
PT_PRE + '*_00_*' + AOI + '*' + 'srp' + '*',
PT_PRE + '*' + AOI + '*' + 'srp' + '*')
else:
repFiles = glob(PT_PRE + '*' + AOI + '*' + 'srp' + '*')
###########################################################################
# Iterate through experiments
###########################################################################
(fNum, digs) = monet.lenAndDigits(repFiles)
fmtStr = '{}+ File: {}/{}'
for (i, repFile) in enumerate(repFiles):
padi = str(i + 1).zfill(digs)
print(fmtStr.format(monet.CBBL, padi, fNum, monet.CEND), end='\r')
(repDta, xpid) = (pkl.load(repFile),
fun.getXpId(repFile, (1, 2, 3, 4, 5, 7)))
xpRow = [da.filterDFWithID(i, xpid) for i in (dfTTI, dfTTO, dfWOP, dfMNX)]
(tti, tto, wop) = [float(row[THS]) for row in xpRow[:3]]
# Pre process
###############################################################################
# print('Diagonal: {}'.format(np.diagonal(psi)))
np.fill_diagonal(psi, 0)
pIn = np.sum(psi.T, axis=0)
# print('ProbIn: {}'.format(pIn))
print('No access: {}'.format(list(np.where(pIn == 0)[0])))
psiN = normalize(psi, axis=1, norm='l2')
# (fig, ax) = pts.plotMatrix(psiN)
# fun.quickSave(fig, ax, PT_IMG, 'transitions.png', dpi=2000)
##############################################################################
# Transitions Matrix and Base Netowrk
##############################################################################
G = nx.from_numpy_matrix(psiN)
G.remove_edges_from(nx.selfloop_edges(G))
G = fun.calcNetworkDistance(G)
##############################################################################
# Community Detection
##############################################################################
coms = cd.markov_clustering(G)
comsID = coms.communities
ptsNum = points.shape[0]
labelsN = [fun.find_in_list_of_list(comsID, i)[0] for i in range(ptsNum)]
##############################################################################
# Geographic Clustering
##############################################################################
kmeans = AgglomerativeClustering(n_clusters=len(comsID)).fit(longlats)
labels = list(kmeans.labels_)
# Reshape to export list of lists of clusters --------------------------------
comsKM = [[] for i in range(len(comsID))]
ids = list(range(ptsNum))