def test_compare_MRST_triangleGrid3D(self):
G_mrst = loadMRSTGrid(
getpath(
"test_gridprocessing/computeGeometry_triangleGrid3D_expected.mat"
))
G_prst = loadMRSTGrid(
getpath("test_gridprocessing/computeGeometry_triangleGrid3D.mat"))
computeGeometry(G_prst)
assert G_mrst == G_prst, G_mrst._cmp(G_prst)
def test_findNeighbors3D(self):
# Tests that computeGeometry finds neighbors properly
# Load grid without computeGeometry from MATLAB
G_prst = loadMRSTGrid(getpath("test_gridprocessing/computeGeometry_findNeighbors3D.mat"))
# ComputeGeometry using PRST
computeGeometry(G_prst)
assert G_prst.cells.volumes.ndim == 2
assert G_prst.faces.areas.ndim == 2
# Load grid WITH computeGeometry from MATLAB
G_mrst = loadMRSTGrid(getpath("test_gridprocessing/computeGeometry_findNeighbors3D_expected.mat"))
assert G_mrst == G_prst, G_mrst._cmp(G_prst)
def __init__(self, station_num, station_name, url, lang, ac_num, ac_name):
self.url = url
self.key = int(station_num.split('/')[1].strip(' >'))
self.ac_num = ac_num
self.ac_name = ac_name
self.station_num = station_num
self.station_name = station_name
self.lang = lang
self.file = None
if lang == ENGLISH:
self.outdir = getpath(MANIPUR_PDF_ENGLISH_DIR)
elif lang == MANIPURI:
self.outdir = getpath(MANIPUR_PDF_MANIPURI_DIR)
else:
raise NotImplementedError
def test_findNeighbors3D(self):
# Tests that computeGeometry finds neighbors properly
# Load grid without computeGeometry from MATLAB
G_prst = loadMRSTGrid(
getpath("test_gridprocessing/computeGeometry_findNeighbors3D.mat"))
# ComputeGeometry using PRST
computeGeometry(G_prst)
assert G_prst.cells.volumes.ndim == 2
assert G_prst.faces.areas.ndim == 2
# Load grid WITH computeGeometry from MATLAB
G_mrst = loadMRSTGrid(
getpath(
"test_gridprocessing/computeGeometry_findNeighbors3D_expected.mat"
))
assert G_mrst == G_prst, G_mrst._cmp(G_prst)
def test_compare_MRST_simple(self):
# G = cartGrid([3 5], [1 1]);
G_mrst = loadMRSTGrid(getpath("test_gridprocessing/cartGrid2D_simple.mat"))
# Using numpy array parameters
G_prst = cartGrid(np.array([3, 5]), np.array([1, 1]))
print(G_mrst.cells.indexMap)
print(G_prst.cells.indexMap)
assert G_mrst == G_prst, G_mrst._cmp(G_prst)
def test_compare_MRST_without_depthz(self):
# Load grid created in Matlab
# G = tensorGrid([1 2 3], [0.5, 1, 1.5], [10, 20])
G_mrst = loadMRSTGrid(getpath("test_gridprocessing/expected_tensorGrid3D_2.mat"))
# Create grid using PRST
x = np.array([1, 2, 3])
y = np.array([0.5, 1, 1.5])
z = np.array([10, 20])
G_prst = tensorGrid(x, y, z)
assert G_mrst == G_prst, G_mrst._cmp(G_prst)
def test_compare_MRST_simple(self):
# Load grid created in Matlab using tensorGrid([0 1 2], [0 1 2])
G_mrst = loadMRSTGrid(getpath("test_gridprocessing/expected_tensorGrid2D_1.mat"))
# Create grid using PRST
x = np.array([0, 1, 2])
y = np.array([0, 1, 2])
G_prst = tensorGrid(x, y)
assert G_mrst == G_prst, G_mrst._cmp(G_prst)
def __init__(self, num, name, loc, telugu, english, ac_num, ac_name,
dist_num, dist_name, session):
self.num = num
self.name = name
self.location = loc
self.telugu = telugu
self.english = english
self.ac_num = ac_num
self.ac_name = ac_name
self.dist_num = dist_num
self.dist_name = dist_name
self.session = session
self.telugu_dir = getpath(ANDHRA_PDF_TELUGU_DIR)
self.english_dir = getpath(ANDHRA_PDF_ENGLISH_DIR)
self.telugu_soup = None
self.english_soup = None
self.telugu_file = None
self.english_file = None
self.download()
def test_compare_MRST_simple(self):
# G = cartGrid([3 5], [1 1]);
G_mrst = loadMRSTGrid(
getpath("test_gridprocessing/cartGrid2D_simple.mat"))
# Using numpy array parameters
G_prst = cartGrid(np.array([3, 5]), np.array([1, 1]))
print(G_mrst.cells.indexMap)
print(G_prst.cells.indexMap)
assert G_mrst == G_prst, G_mrst._cmp(G_prst)
def test_compare_MRST_with_depthz(self):
# Load grid created in Matlab
# G = tensorGrid([1 2 3], [0.5, 1, 1.5], [10, 20], 'depthz', [ 1 2 3; 4 5 6; 7 8 9]);
G_mrst = loadMRSTGrid(getpath("test_gridprocessing/expected_tensorGrid3D_1.mat"))
# Create grid using PRST
x = np.array([1, 2, 3])
y = np.array([0.5, 1, 1.5])
z = np.array([10, 20])
depthz = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
G_prst = tensorGrid(x, y, z, depthz=depthz)
assert G_mrst == G_prst, G_mrst._cmp(G_prst)
def test_compare_MRST_simple(self):
# Load grid created in Matlab using tensorGrid([0 1 2], [0 1 2])
G_mrst = loadMRSTGrid(
getpath("test_gridprocessing/expected_tensorGrid2D_1.mat"))
# Create grid using PRST
x = np.array([0, 1, 2])
y = np.array([0, 1, 2])
G_prst = tensorGrid(x, y)
assert G_mrst == G_prst, G_mrst._cmp(G_prst)
def test_compare_MRST_without_depthz(self):
# Load grid created in Matlab
# G = tensorGrid([1 2 3], [0.5, 1, 1.5], [10, 20])
G_mrst = loadMRSTGrid(
getpath("test_gridprocessing/expected_tensorGrid3D_2.mat"))
# Create grid using PRST
x = np.array([1, 2, 3])
y = np.array([0.5, 1, 1.5])
z = np.array([10, 20])
G_prst = tensorGrid(x, y, z)
assert G_mrst == G_prst, G_mrst._cmp(G_prst)
def test_compare_MRST_with_depthz(self):
# Load grid created in Matlab
# G = tensorGrid([1 2 3], [0.5, 1, 1.5], [10, 20], 'depthz', [ 1 2 3; 4 5 6; 7 8 9]);
G_mrst = loadMRSTGrid(
getpath("test_gridprocessing/expected_tensorGrid3D_1.mat"))
# Create grid using PRST
x = np.array([1, 2, 3])
y = np.array([0.5, 1, 1.5])
z = np.array([10, 20])
depthz = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
G_prst = tensorGrid(x, y, z, depthz=depthz)
assert G_mrst == G_prst, G_mrst._cmp(G_prst)
def test_gravityColumn(self):
G = cartGrid([1,1,30])
computeGeometry(G)
rock = Rock(G, perm=0.1*darcy, poro=1)
rock.perm[0:G.cells.num//2,:] = 0.2*darcy
T = computeTrans(G, rock)
mrst_T = loadmat(getpath("test_solvers/computeTrans_gravityColumn_T.mat"),
squeeze_me=True, struct_as_record=False)["T"]
if mrst_T.ndim == 1:
mrst_T = mrst_T[:,np.newaxis]
assert np.array_equal(T.shape, mrst_T.shape)
assert np.allclose(T, mrst_T)
def test_gravityColumn(self):
G = cartGrid([1, 1, 30])
computeGeometry(G)
rock = Rock(G, perm=0.1 * darcy, poro=1)
rock.perm[0:G.cells.num // 2, :] = 0.2 * darcy
T = computeTrans(G, rock)
mrst_T = loadmat(
getpath("test_solvers/computeTrans_gravityColumn_T.mat"),
squeeze_me=True,
struct_as_record=False)["T"]
if mrst_T.ndim == 1:
mrst_T = mrst_T[:, np.newaxis]
assert np.array_equal(T.shape, mrst_T.shape)
assert np.allclose(T, mrst_T)
def log_configurer():
root = logging.getLogger()
root.setLevel(logging.WARNING)
formatter = logging.Formatter(fmt='%(message)s')
consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(formatter)
fileHandler = RotatingFileHandler(getpath(LOG_FILE),
maxBytes=MAX_LOG_SIZE,
backupCount=LOG_BACKUP_COUNT)
fileHandler.setFormatter(formatter)
root.addHandler(consoleHandler)
root.addHandler(fileHandler)
return root
def download(self):
try:
js = urlget(self.js_url)
except AssertionError:
return
self.js_content = js
parsed = self.parse_js()
for item in parsed:
ac, polls = item
ac_num, ac_name = ac
for i in range(polls):
poll_num = i + 1
url = self.get_pdf_url(ac_num, poll_num)
try:
file = urldown(url=url, dest=getpath(TRIPURA_PDF_DIR))
except AssertionError:
file = None
row = (self.name, str(ac_num), ac_name, str(poll_num),
relpath(file) if file is not None else
'Not available / Unable to download')
self.rolls.append(row)
append_csv(OUTPUT_FILE, row)
def test_gravityColumn(self):
import numpy as np
import prst
import prst.incomp as incomp
import prst.gridprocessing as gridprocessing
import prst.utils as utils
import prst.params as params
import prst.solvers as solvers
from prst.utils.units import centi, poise, kilogram, meter, bar, darcy
prst.gravity_reset()
G = gridprocessing.cartGrid([1, 1, 30], [1, 1, 30])
gridprocessing.computeGeometry(G)
rock = params.rock.Rock(G, perm=0.1*darcy, poro=1)
fluid = incomp.fluid.SingleFluid(viscosity=1*centi*poise,
density=1014*kilogram/meter**3)
bc = params.wells_and_bc.BoundaryCondition()
bc.addPressureSide(G, "top", 100*bar)
T = solvers.computeTrans(G, rock)
resSol = solvers.initResSol(G, p0=0.0)
psol = incomp.incompTPFA(resSol, G, T, fluid, bc=bc)
# Load MRST results and compare solution pressure, flux, saturation,
# facePressure.
matfile = getpath("test_example_gravityColumn/sol.mat")
msol = loadmat(matfile, squeeze_me=True, struct_as_record=False)["sol"]
msol_pressure = np.atleast_2d(msol.pressure).transpose()
msol_flux = np.atleast_2d(msol.flux).transpose()
msol_s = np.atleast_2d(msol.s).transpose()
msol_facePressure = np.atleast_2d(msol.facePressure).transpose()
assert np.allclose(psol.pressure, msol_pressure, rtol=1e-11)
assert np.allclose(psol.flux, msol_flux, rtol=1e-11)
assert np.allclose(psol.s, msol_s, rtol=1e-11)
assert np.allclose(psol.facePressure, msol_facePressure, rtol=1e-11)
def test_array_shapes(self):
G = loadMRSTGrid(getpath("test_io/expected_tensorGrid2D_1G.mat"))
assert G.cells.facePos.shape[1] == 1
assert G.cells.indexMap.shape[1] == 1
from bs4 import BeautifulSoup
from helpers import urlget, getpath, relpath, baseurl, urljoin, urldown, \
timestamp, TRACK_FILE_TS, append_csv
ENGLISH = 'English'
MANIPURI = 'Manipuri'
MANIPUR_PDF_ENGLISH_DIR = 'manipur_pdfs/english'
MANIPUR_PDF_MANIPURI_DIR = 'manipur_pdfs/manipuri'
MANIPUR_TRACK_DIR = './'
ENGLISH_URL = 'http://www.ceomanipur.nic.in/ElectoralRolls/ElectoralRolls_English.html'
MANIPURI_URL = 'http://www.ceomanipur.nic.in/ElectoralRolls/ElectoralRolls_Manipuri.html'
CSV_HEADER = ('ac_number', 'ac_name', 'poll_station_number',
'poll_station_name', 'language', 'relative_path')
OUTPUT_FILE = getpath(MANIPUR_TRACK_DIR,
'Manipur-%s.csv' % timestamp(TRACK_FILE_TS))
class Manipur:
def __init__(self):
self.rolls = [{
'url': ENGLISH_URL,
'lang': ENGLISH,
'html': None,
'data': []
}, {
'url': MANIPURI_URL,
'lang': MANIPURI,
'html': None,
'data': []
}]
def test_tensorGrid2D(self):
G = loadMRSTGrid(getpath("test_io/expected_tensorGrid2D_1G.mat"))
# Check existence of top-level attributes
assert hasattr(G, "cells")
assert hasattr(G, "faces")
assert hasattr(G, "nodes")
assert hasattr(G, "cartDims")
assert hasattr(G, "gridType")
# Cells
assert hasattr(G.cells, "num")
assert G.cells.num == 4
assert np.array_equal(G.cells.facePos, np.array(
[[0], [4], [8], [12], [16]]))
assert np.array_equal(G.cells.indexMap, np.array(
[[0], [1], [2], [3]]))
assert np.array_equal(G.cells.faces, np.array([
[0, 0],
[6, 2],
[1, 1],
[8, 3],
[1, 0],
[7, 2],
[2, 1],
[9, 3],
[3, 0],
[8, 2],
[4, 1],
[10, 3],
[4, 0],
[9, 2],
[5, 1],
[11, 3],
]))
# Faces
assert G.faces.num == 12
assert np.array_equal(G.faces.nodePos, np.array([
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24
])[:,np.newaxis])
# This array is saved as uint8, and if not carefully loaded, -1 will
# become 255.
assert G.faces.neighbors[0,0] == -1
assert np.array_equal(G.faces.neighbors, np.array([
[-1, 0],
[0, 1],
[1, -1],
[-1, 2],
[2, 3],
[3, -1],
[-1, 0],
[-1, 1],
[0, 2],
[1, 3],
[2, -1],
[3, -1],
]))
# G.faces.tag is not available in PRST
assert np.array_equal(G.faces.nodes, np.array([
0, 3, 1, 4, 2, 5, 3, 6, 4, 7, 5, 8, 1, 0, 2, 1, 4, 3, 5, 4, 7,
6, 8, 7
])[:,np.newaxis])
# Nodes
assert G.nodes.num == 9
# G.nodes.coords
assert np.array_equal(G.nodes.coords, np.array([
[0, 0],
[1, 0],
[2, 0],
[0, 1],
[1, 1],
[2, 1],
[0, 2],
[1, 2],
[2, 2],
]))
# Top-level attribues
# cartDims
assert G.cartDims[0] == 2 and G.cartDims[1] == 2
assert "tensorGrid" in G.gridType
assert G.gridType == ["tensorGrid"]
assert G.gridDim == 2
def test_no_cartDims(self):
G = loadMRSTGrid(getpath("test_io/grid_without_indexMap_or_cartDims.mat"), "V")
assert not hasattr(G, "cartDims")
def test_compare_MRST_triangleGrid3D(self):
G_mrst = loadMRSTGrid(getpath("test_gridprocessing/computeGeometry_triangleGrid3D_expected.mat"))
G_prst = loadMRSTGrid(getpath("test_gridprocessing/computeGeometry_triangleGrid3D.mat"))
computeGeometry(G_prst)
assert G_mrst == G_prst, G_mrst._cmp(G_prst)
def test_findNeighbors2D(self):
G_prst = loadMRSTGrid(
getpath("test_gridprocessing/computeGeometry_findNeighbors2D.mat"))
with pytest.raises(ValueError):
computeGeometry(G_prst)
def test_equal_without_indexMap(self):
V = loadMRSTGrid(
getpath("test_gridprocessing/grid_equal_without_indexMap.mat"),
"V")
V == V
REFRESH_DELAY = 10 # seconds
REFRESH_REQUEST_TIMEOUT = 30 # seconds
REFRESH_TOTAL_RETRIES = 3
DELAY_MIN = 0 # seconds
DELAY_MAX = 0 # seconds
DELAY_FRACTION = 999999
# Connection settings
USER_AGENT = 'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0'
ANDHRA_URL = 'http://ceoaperms.ap.gov.in/electoral_rolls/Rolls.aspx'
ANDHRA_BASE_URL = 'http://ceoaperms.ap.gov.in/electoral_rolls/'
# Parsing & saving
FIND_PDF_REGEX = re.compile(r"open\('(.+)',")
OUTPUT_FILE = getpath(
ANDHRA_TRACK_DIR, 'Andhra{}-{}.csv'.format(ASSIGNED_ID or '',
timestamp(TRACK_FILE_TS)))
DOWNLOAD_FAILED = 'Not available / Unable to download'
TRACK_FILE = getpath('cache/andhra{}_track.bin'.format(ASSIGNED_ID or ''))
CSV_HEADER = ('district_name', 'ac_name', 'polling_station_number',
'polling_station_name', 'polling_station_location',
'telugu_file_name', 'eng_file_name')
# Log settings
MAX_LOG_SIZE = 52428800
LOG_BACKUP_COUNT = 5
LOG_FILE = getpath('logs/andhra{}.log'.format(ASSIGNED_ID or ''))
def log_configurer():
root = logging.getLogger()
def test_single_grid_type(self):
G = loadMRSTGrid(getpath("test_io/expected_tensorGrid2D_1G.mat"))
assert isinstance(G.gridType, list)
def test_tensorGrid2D_V(self):
V = loadMRSTGrid(getpath("test_io/expected_tensorGrid2D_1V.mat"), "V")
assert V.cells.num == 4
def test_findNeighbors3D_force(self):
G_mrst = loadMRSTGrid(
getpath(
"test_gridprocessing/computeGeometry_findNeighbors3D_expected.mat"
))
computeGeometry(G_mrst, findNeighbors=True)
def test_findNeighbors3D_force(self):
G_mrst = loadMRSTGrid(getpath("test_gridprocessing/computeGeometry_findNeighbors3D_expected.mat"))
computeGeometry(G_mrst, findNeighbors=True)
def test_compare_MRST_simple(self):
# G = cartGrid([3 5 7], [1 1 3]);
G_mrst = loadMRSTGrid(getpath("test_gridprocessing/cartGrid3D_simple.mat"))
G_prst = cartGrid(np.array([3, 5, 7]), np.array([1, 1, 3]))
assert G_mrst == G_prst, G_mrst._cmp(G_prst)
def test_findNeighbors2D(self):
G_prst = loadMRSTGrid(getpath("test_gridprocessing/computeGeometry_findNeighbors2D.mat"))
with pytest.raises(ValueError):
computeGeometry(G_prst)
def test_compare_MRST_simple(self):
# G = cartGrid([3 5 7], [1 1 3]);
G_mrst = loadMRSTGrid(
getpath("test_gridprocessing/cartGrid3D_simple.mat"))
G_prst = cartGrid(np.array([3, 5, 7]), np.array([1, 1, 3]))
assert G_mrst == G_prst, G_mrst._cmp(G_prst)
def test_computeGeometry_shapes(self):
G = loadMRSTGrid(getpath("test_gridprocessing/computeGeometry_findNeighbors3D_expected.mat"))
assert G.cells.centroids.ndim == 2
import logging
import sys
import prst
from prst.io import loadMRSTGrid
from helpers import getpath
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
print("yah")
print(getpath("test_gridprocessing/grid_equal_without_indexMap.mat"))
V = loadMRSTGrid(getpath("test_gridprocessing/grid_equal_without_indexMap.mat"), "V")
print("duew")
V == V
print("wat")
def test_multiple_grid_types(self):
G = loadMRSTGrid(getpath("test_io/multiple_gridtypes.mat"))
assert isinstance(G.gridType, list)
def test_malformed_gridType(self):
with pytest.raises(ValueError):
G = loadMRSTGrid(getpath("test_io/malformed_gridType.mat"))
def test_equal_without_indexMap(self):
V = loadMRSTGrid(getpath("test_gridprocessing/grid_equal_without_indexMap.mat"), "V")
V == V
import logging
import sys
import prst
from prst.io import loadMRSTGrid
from helpers import getpath
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
print("yah")
print(getpath("test_gridprocessing/grid_equal_without_indexMap.mat"))
V = loadMRSTGrid(
getpath("test_gridprocessing/grid_equal_without_indexMap.mat"), "V")
print("duew")
V == V
print("wat")
#!/usr/bin/env python3
from helpers import urlget, urldown, urljoin, getpath, relpath, \
timestamp, TRACK_FILE_TS, append_csv
TRIPURA_PDF_DIR = 'tripura_pdfs'
TRIPURA_TRACK_DIR = './'
WEST_JS_URL = 'http://ceotripura.nic.in/PSCDROM/Draft_2018/PC01/html/js/list.js'
WEST_BASE_URL = 'http://ceotripura.nic.in/PSCDROM/Draft_2018/PC01/'
EAST_JS_URL = 'http://ceotripura.nic.in/PSCDROM/Draft_2018/PC02/html/js/list.js'
EAST_BASE_URL = 'http://ceotripura.nic.in/PSCDROM/Draft_2018/PC02/'
CSV_HEADER = ('district_name', 'ac_number', 'ac_name', 'poll_station_number',
'filename')
OUTPUT_FILE = getpath(TRIPURA_TRACK_DIR,
'Tripura-%s.csv' % timestamp(TRACK_FILE_TS))
class Tripura:
def __init__(self):
self.districts = [
District(js_url=WEST_JS_URL, base_url=WEST_BASE_URL),
District(js_url=EAST_JS_URL, base_url=EAST_BASE_URL)
]
def download(self):
# Write CSV header
append_csv(OUTPUT_FILE, CSV_HEADER)
# Run download
for district in self.districts:
district.download()
