def prodstat_option_matrix(
ranges: Union[ProdStatRange, List[ProdStatRange]],
months: Optional[Union[int, List[int]]],
include_zeroes: Union[bool, List[bool]] = [True, False],
) -> List[Tuple[ProdStatRange, int, bool]]:
return list(
itertools.product(
util.ensure_list(ranges),
util.ensure_list(months),
util.ensure_list(include_zeroes),
))
def production_monthly(self) -> List:
output: List = []
years: Dict = {}
# logger.debug(
# f"{self.production_header.get('api14')} -- processing production records"
# )
if hasattr(self, "production"):
years = self.production.get("year")
if years:
# logger.debug(f"years: {[x.get('number') for x in ensure_list(years)]}")
for year in ensure_list(years):
yr = year.get("number")
# logger.debug(
# f"{self.production_header.get('api14')} -- processing prod year {yr}"
# )
for month in ensure_list(year.get("month", {})):
out = {}
get = functools.partial(query_dict, data=month)
mo = month.get("number", None)
last_day = month.get("last_day", None)
out["year"] = yr
out["month"] = mo
out["last_day"] = last_day
out["first_date"] = datetime(year=yr, month=mo, day=1)
out["last_date"] = datetime(year=yr,
month=mo,
day=last_day)
out["liquid"] = get("total_liquid.value")
out["liquid_uom"] = get("total_liquid.uom")
# out["oil"] = get("oil.value")
# out["oil_uom"] = get("oil.uom")
out["gas"] = get("total_gas.value")
out["gas_uom"] = get("total_gas.uom")
out["casinghead_gas"] = get("casinghead_gas.value")
out["casinghead_gas_uom"] = get("casinghead_gas.uom")
out["water"] = get("water.value")
out["water_uom"] = get("water.uom")
out["gor"] = get("ratios.gas_oil.value")
out["gor_uom"] = get("ratios.gas_oil.uom")
out["water_cut"] = get("ratios.water_cut")
out["well_count"] = get("wells.total")
out["oil_well_count"] = get("wells.oil")
output.append(out)
return output
def before_request():
g.start = time.time()
request.id = shortuuid.uuid()
request.should_log = random.random() < conf.WEB_LOG_SAMPLE_FRAC # pairs request/response logs # noqa
request.arg_counts = {
k: len(ensure_list(v)) for k, v in (request.args or {}).items()
}
request.arg_count_str = " ".join(
[f" {k}s={v}" for k, v in request.arg_counts.items()]
)
if conf.WEB_LOG_REQUESTS:
attrs = {
"request": {
"request_at": utcnow().strftime("%d/%b/%Y:%H:%M:%S.%f")[:-3],
"remote_addr": request.remote_addr,
"method": request.method,
"path": request.path,
"query_string": request.query_string,
"scheme": request.scheme,
"referrer": request.referrer,
"user_agent": request.user_agent,
"headers": request.headers,
"args": request.args,
},
}
if request.should_log:
logger.info(
f"[{request.id}] {request.method} - {request.scheme}:{request.path}{request.arg_count_str}", # noqa
extra=attrs,
)
def get_active_survey(cls, data: OrderedDict) -> Union[OrderedDict, None]:
"""Return the most recent survey (survey with the highest 'number')"""
active_survey = None
number = 0
for s in ensure_list(data.get("surveys", {})):
get = functools.partial(query_dict, data=s)
n = get("borehole.header.number") or -1
if n > number:
active_survey = get("borehole")
number = n
return active_survey
async def download(
self,
api14s: Union[str, List[str]] = None,
api10s: Union[str, List[str]] = None,
entities: Union[str, List[str]] = None,
entity12s: Union[str, List[str]] = None,
**kwargs,
) -> DataSet:
kwargs = {**self.download_kwargs, **kwargs}
try:
ts = timer()
# TODO: Move to router
if self.hole_dir == HoleDirection.H:
path = IHSPath.prod_h
elif self.hole_dir == HoleDirection.V:
path = IHSPath.prod_v
else:
raise ValueError("Cant determine request path")
prodset: ProdSet = await pd.DataFrame.prodstats.from_ihs(
path=path,
api14s=api14s,
api10s=api10s,
entities=entities,
entity12s=entity12s,
**kwargs,
)
exc_time = round(timer() - ts, 2)
self.add_metric(
operation="download",
name="*",
seconds=exc_time,
count=prodset.header.shape[0],
)
return prodset
except Exception as e:
self.raise_execution_error(
operation="download",
record_count=len(
util.ensure_list(entities or entity12s or api14s or api10s
or [])),
e=e,
extra={
"entities": entities,
"entity12s": entity12s,
"api14s": api14s,
"api10s": api10s,
},
)
raise e
def time_solve(grid):
start = time.clock()
_values = solve(grid, multi=multi)
all_values = ensure_list(_values)
t = time.clock() - start
# Display puzzles that take long enough
if (show_min_time is not None
and t > show_min_time) or (show_multi and len(all_values) > 1):
display(grid_values(grid))
for values in all_values:
if values:
display(values)
print '(%.2f seconds)\n' % t
return grid, t, filter(solved, all_values)
def get_config():
parser = argparse.ArgumentParser(
description='Detect cheat in vjudge contest.')
# basic configuration
parser.add_argument('-c',
'--config',
dest='config_file',
help='YAML config file path')
config = parser.parse_args()
if config.config_file is not None:
with open(config.config_file, "r", encoding='utf-8') as f:
y = yaml.load(f)
config.ignore_userid = util.ensure_list(y.get("ignore_userid", []))
config.include_userid = util.ensure_list(
y.get("include_userid", []))
if len(config.include_userid) == 0:
config.include_userid.append(".+")
config.include_userid = list(
map(lambda x: re.compile(f"^{x}$"), config.include_userid))
config.semester = y.get("semester")
print(config)
return config
async def get_jobs(
cls,
api14s: Union[str, List[str]] = None,
api10s: Union[str, List[str]] = None,
params: Dict = None,
timeout: Optional[int] = None,
concurrency: int = None,
**kwargs,
) -> List[Dict[str, Any]]:
if api14s and api10s:
raise ValueError(
"Exactly one of [api14s, api10s] can be specified")
if api14s:
path = FracFocusPath.api14
ids = api14s
elif api10s:
path = FracFocusPath.api10
ids = api10s
else:
raise ValueError("One of [api14s, api10s] must be specified")
responses: List[httpx.Response] = []
ids = util.ensure_list(ids)
concurrency = concurrency or 50
params = params or {}
async with cls(**kwargs) as client:
coros: List[Coroutine] = []
for id in ids:
coro = client.get(f"{path.value}/{id}", timeout=timeout)
coros.append(coro)
for idx, chunk in enumerate(util.chunks(coros, concurrency)):
responses += await asyncio.gather(*chunk)
data: List[Dict[str, Any]] = []
for r in responses:
json: Dict = r.json() # type: ignore
if "data" in json.keys():
data += json["data"]
return data
async def download(
self,
api14s: Union[str, List[str]] = None,
api10s: Union[str, List[str]] = None,
**kwargs,
) -> WellGeometrySet:
kwargs = {**self.download_kwargs, **kwargs}
try:
ts = timer()
# TODO: Move to router
if self.hole_dir == HoleDirection.H:
path = IHSPath.well_h_geoms
elif self.hole_dir == HoleDirection.V:
path = IHSPath.well_v_geoms
else:
raise ValueError("Cant determine request path")
geoms: WellGeometrySet = await pd.DataFrame.shapes.from_ihs(
path=path,
api14s=api14s,
api10s=api10s,
**kwargs,
)
exc_time = round(timer() - ts, 2)
self.add_metric(
operation="download",
name="*",
seconds=exc_time,
count=geoms.locations.shape[0],
)
return geoms
except Exception as e:
count = len(util.ensure_list(api10s or api14s))
self.raise_execution_error(
operation="download",
record_count=count,
e=e,
extra={
"api10s": api10s,
"api14s": api14s
},
)
async def download(
self,
api14s: Union[str, List[str]] = None,
api10s: Union[str, List[str]] = None,
**kwargs,
) -> WellSet:
kwargs = {**self.download_kwargs, **kwargs}
try:
ts = timer()
# TODO: Add sample option
wellset = await pd.DataFrame.wells.from_multiple(
hole_dir=self.hole_dir,
api14s=api14s,
api10s=api10s,
**kwargs,
)
exc_time = round(timer() - ts, 2)
self.add_metric(
operation="download",
name="*",
seconds=exc_time,
count=wellset.wells.shape[0],
)
return wellset
except Exception as e:
count = len(util.ensure_list(api10s or api14s))
self.raise_execution_error(
operation="download",
record_count=count,
e=e,
extra={
"api10s": api10s,
"api14s": api14s
},
)
async def agg(
self,
funcs: Union[Function, List[Function]],
filter: Union[str, TextClause, BinaryExpression] = None,
) -> Dict[str, Union[int, float]]:
func_map: Dict[str, Function] = {
f.name: f
for f in util.ensure_list(funcs)
}
stmt = self.model.select(*func_map.values())
if filter is not None:
if not isinstance(filter, (TextClause, BinaryExpression)):
filter = text(filter)
stmt = stmt.where(filter)
result: db.Row
async with db.Session() as session:
async with session.begin():
result = (await session.execute(stmt)).one()
return dict(zip(func_map, result))
async def _get(
cls,
ids: Union[str, List[str]],
path: IHSPath,
param_name: str,
params: Dict = None,
timeout: Optional[int] = None,
concurrency: int = None,
**kwargs,
) -> List[Dict[str, Any]]:
responses: List[httpx.Response] = []
ids = util.ensure_list(ids)
concurrency = concurrency or 50
timeout = timeout or 300
params = params or {}
async with cls(**kwargs) as client:
coros: List[Coroutine] = []
for id in ids:
coro = client.get(
path.value, params={param_name: id, **params}, timeout=timeout,
)
coros.append(coro)
for idx, chunk in enumerate(util.chunks(coros, concurrency)):
responses += await asyncio.gather(*chunk)
data: List[Dict[str, Any]] = []
for r in responses:
json: Dict = r.json() # type: ignore
if "data" in json.keys():
data += json["data"]
return data
def get_transforms(self):
if self.init_transform is None:
return util.ensure_list(self.transform)
return [self.transform, self.init_transform]
import sqlite3
import util
if __name__ == "__main__":
os.nice(19)
util.setup("temp/", "")
all_filepaths = ['brainSegmentation.db']
conn = sqlite3.connect(util.DB_PATH)
conn.text_factory = str
cursor = conn.execute('''SELECT filepath, transform, filepath_reg from Images''')
for (filepath, transform, filepath_reg) in cursor:
all_filepaths.extend(util.ensure_list(filepath))
all_filepaths.extend(util.ensure_list(filepath_reg))
if transform is None:
continue
for _transform in transform.split(","):
all_filepaths.append(_transform.strip())
cursor = conn.execute('''SELECT filepath, filepath_reg from Labels''')
for (filepath, filepath_reg) in cursor:
all_filepaths.extend(util.ensure_list(filepath))
all_filepaths.extend(util.ensure_list(filepath_reg))
for root, dirs, files in os.walk(util.DATA_FOLDER):
for filepath in files:
filepath = os.path.join(root, filepath).replace(util.DATA_FOLDER, "")
if filepath not in all_filepaths:
def get_config():
parser = argparse.ArgumentParser(
description='Detect cheat in vjudge contest.')
# basic configuration
parser.add_argument(dest='files',
type=str,
nargs='+',
help='zip files of vjudge contest submission')
parser.add_argument('-c',
'--config',
dest='config_file',
help='YAML config file path')
parser.add_argument('-l',
dest='language',
nargs='+',
default=['cc'],
help='language to detect')
parser.add_argument(
'-a',
dest='add_dir',
help=
'additional files to detect cheating, put them under ${add_dir}/${problem_id}'
)
# preprocess configuration
parser.add_argument('-r',
dest='remove_duplicate',
action='store_true',
help='remove duplicate submissions of a user')
parser.add_argument(
'-u',
dest='include_userid',
nargs='+',
default=[],
help='regex of user id to be included, include all if not specified')
parser.add_argument(
'-iu',
dest='ignore_userid',
nargs='+',
default=[],
help='user id to be ignored, has higher priority than include')
parser.add_argument('-ip',
dest='ignore_problem',
nargs='+',
default=[],
help='problem to be ignored')
# moss configuration
parser.add_argument(
'-id',
dest='moss_userid',
help='moss userid, regiester one from e-mail if you don\'t have one')
parser.add_argument(
'-b',
dest='base_dir',
help='use in moss, put base files under ${base_dir}/${problem_id}/')
parser.add_argument(
'-m',
dest='maximal_match',
type=int,
default=0,
help=
'use in moss, maximal appearance before identified as code in base file'
)
parser.add_argument('-n',
dest='report_number',
type=int,
default=0,
help='use in moss, maximal item number in report')
config = parser.parse_args()
if config.config_file is not None:
with open(config.config_file, "r", encoding='utf-8') as f:
y = yaml.load(f)
config.moss_userid = config.moss_userid or y.get("moss_userid")
config.remove_duplicate = config.remove_duplicate or y.get(
"remove_duplicate")
config.maximal_match = config.maximal_match or y.get(
"maximal_match") or 10
config.report_number = config.report_number or y.get(
"report_number") or 250
config.ignore_userid.extend(
util.ensure_list(y.get("ignore_userid", [])))
config.language.extend(util.ensure_list(y.get("language", [])))
config.include_userid.extend(
util.ensure_list(y.get("include_userid", [])))
if len(config.include_userid) == 0:
config.include_userid.append(".+")
config.include_userid = list(
map(lambda x: re.compile(f"^{x}$"), config.include_userid))
return config
def get_transforms(self):
if self.init_transform is None:
return util.ensure_list(self.transform)
return [self.transform, self.init_transform]
def test_ensure_list(self, data, expected):
assert util.ensure_list(data) == expected
REMOVE = False
if __name__ == "__main__":
os.nice(19)
util.setup("temp/")
all_filepaths = ['brainSegmentation.db']
conn = sqlite3.connect(util.DB_PATH)
conn.text_factory = str
cursor = conn.execute(
'''SELECT filepath, transform, filepath_reg from Images''')
for (filepath, transform, filepath_reg) in cursor:
all_filepaths.extend(util.ensure_list(filepath))
all_filepaths.extend(util.ensure_list(filepath_reg))
if transform is None:
continue
for _transform in transform.split(","):
all_filepaths.append(_transform.strip())
cursor = conn.execute('''SELECT filepath, filepath_reg from Labels''')
for (filepath, filepath_reg) in cursor:
all_filepaths.extend(util.ensure_list(filepath))
all_filepaths.extend(util.ensure_list(filepath_reg))
for root, dirs, files in os.walk(util.DATA_FOLDER):
for filepath in files:
filepath = os.path.join(root,
filepath).replace(util.DATA_FOLDER, "")
