def prodstat_option_matrix( ranges: Union[ProdStatRange, List[ProdStatRange]], months: Optional[Union[int, List[int]]], include_zeroes: Union[bool, List[bool]] = [True, False], ) -> List[Tuple[ProdStatRange, int, bool]]: return list( itertools.product( util.ensure_list(ranges), util.ensure_list(months), util.ensure_list(include_zeroes), ))
def production_monthly(self) -> List: output: List = [] years: Dict = {} # logger.debug( # f"{self.production_header.get('api14')} -- processing production records" # ) if hasattr(self, "production"): years = self.production.get("year") if years: # logger.debug(f"years: {[x.get('number') for x in ensure_list(years)]}") for year in ensure_list(years): yr = year.get("number") # logger.debug( # f"{self.production_header.get('api14')} -- processing prod year {yr}" # ) for month in ensure_list(year.get("month", {})): out = {} get = functools.partial(query_dict, data=month) mo = month.get("number", None) last_day = month.get("last_day", None) out["year"] = yr out["month"] = mo out["last_day"] = last_day out["first_date"] = datetime(year=yr, month=mo, day=1) out["last_date"] = datetime(year=yr, month=mo, day=last_day) out["liquid"] = get("total_liquid.value") out["liquid_uom"] = get("total_liquid.uom") # out["oil"] = get("oil.value") # out["oil_uom"] = get("oil.uom") out["gas"] = get("total_gas.value") out["gas_uom"] = get("total_gas.uom") out["casinghead_gas"] = get("casinghead_gas.value") out["casinghead_gas_uom"] = get("casinghead_gas.uom") out["water"] = get("water.value") out["water_uom"] = get("water.uom") out["gor"] = get("ratios.gas_oil.value") out["gor_uom"] = get("ratios.gas_oil.uom") out["water_cut"] = get("ratios.water_cut") out["well_count"] = get("wells.total") out["oil_well_count"] = get("wells.oil") output.append(out) return output
def before_request(): g.start = time.time() request.id = shortuuid.uuid() request.should_log = random.random() < conf.WEB_LOG_SAMPLE_FRAC # pairs request/response logs # noqa request.arg_counts = { k: len(ensure_list(v)) for k, v in (request.args or {}).items() } request.arg_count_str = " ".join( [f" {k}s={v}" for k, v in request.arg_counts.items()] ) if conf.WEB_LOG_REQUESTS: attrs = { "request": { "request_at": utcnow().strftime("%d/%b/%Y:%H:%M:%S.%f")[:-3], "remote_addr": request.remote_addr, "method": request.method, "path": request.path, "query_string": request.query_string, "scheme": request.scheme, "referrer": request.referrer, "user_agent": request.user_agent, "headers": request.headers, "args": request.args, }, } if request.should_log: logger.info( f"[{request.id}] {request.method} - {request.scheme}:{request.path}{request.arg_count_str}", # noqa extra=attrs, )
def get_active_survey(cls, data: OrderedDict) -> Union[OrderedDict, None]: """Return the most recent survey (survey with the highest 'number')""" active_survey = None number = 0 for s in ensure_list(data.get("surveys", {})): get = functools.partial(query_dict, data=s) n = get("borehole.header.number") or -1 if n > number: active_survey = get("borehole") number = n return active_survey
async def download( self, api14s: Union[str, List[str]] = None, api10s: Union[str, List[str]] = None, entities: Union[str, List[str]] = None, entity12s: Union[str, List[str]] = None, **kwargs, ) -> DataSet: kwargs = {**self.download_kwargs, **kwargs} try: ts = timer() # TODO: Move to router if self.hole_dir == HoleDirection.H: path = IHSPath.prod_h elif self.hole_dir == HoleDirection.V: path = IHSPath.prod_v else: raise ValueError("Cant determine request path") prodset: ProdSet = await pd.DataFrame.prodstats.from_ihs( path=path, api14s=api14s, api10s=api10s, entities=entities, entity12s=entity12s, **kwargs, ) exc_time = round(timer() - ts, 2) self.add_metric( operation="download", name="*", seconds=exc_time, count=prodset.header.shape[0], ) return prodset except Exception as e: self.raise_execution_error( operation="download", record_count=len( util.ensure_list(entities or entity12s or api14s or api10s or [])), e=e, extra={ "entities": entities, "entity12s": entity12s, "api14s": api14s, "api10s": api10s, }, ) raise e
def time_solve(grid): start = time.clock() _values = solve(grid, multi=multi) all_values = ensure_list(_values) t = time.clock() - start # Display puzzles that take long enough if (show_min_time is not None and t > show_min_time) or (show_multi and len(all_values) > 1): display(grid_values(grid)) for values in all_values: if values: display(values) print '(%.2f seconds)\n' % t return grid, t, filter(solved, all_values)
def get_config(): parser = argparse.ArgumentParser( description='Detect cheat in vjudge contest.') # basic configuration parser.add_argument('-c', '--config', dest='config_file', help='YAML config file path') config = parser.parse_args() if config.config_file is not None: with open(config.config_file, "r", encoding='utf-8') as f: y = yaml.load(f) config.ignore_userid = util.ensure_list(y.get("ignore_userid", [])) config.include_userid = util.ensure_list( y.get("include_userid", [])) if len(config.include_userid) == 0: config.include_userid.append(".+") config.include_userid = list( map(lambda x: re.compile(f"^{x}$"), config.include_userid)) config.semester = y.get("semester") print(config) return config
async def get_jobs( cls, api14s: Union[str, List[str]] = None, api10s: Union[str, List[str]] = None, params: Dict = None, timeout: Optional[int] = None, concurrency: int = None, **kwargs, ) -> List[Dict[str, Any]]: if api14s and api10s: raise ValueError( "Exactly one of [api14s, api10s] can be specified") if api14s: path = FracFocusPath.api14 ids = api14s elif api10s: path = FracFocusPath.api10 ids = api10s else: raise ValueError("One of [api14s, api10s] must be specified") responses: List[httpx.Response] = [] ids = util.ensure_list(ids) concurrency = concurrency or 50 params = params or {} async with cls(**kwargs) as client: coros: List[Coroutine] = [] for id in ids: coro = client.get(f"{path.value}/{id}", timeout=timeout) coros.append(coro) for idx, chunk in enumerate(util.chunks(coros, concurrency)): responses += await asyncio.gather(*chunk) data: List[Dict[str, Any]] = [] for r in responses: json: Dict = r.json() # type: ignore if "data" in json.keys(): data += json["data"] return data
async def download( self, api14s: Union[str, List[str]] = None, api10s: Union[str, List[str]] = None, **kwargs, ) -> WellGeometrySet: kwargs = {**self.download_kwargs, **kwargs} try: ts = timer() # TODO: Move to router if self.hole_dir == HoleDirection.H: path = IHSPath.well_h_geoms elif self.hole_dir == HoleDirection.V: path = IHSPath.well_v_geoms else: raise ValueError("Cant determine request path") geoms: WellGeometrySet = await pd.DataFrame.shapes.from_ihs( path=path, api14s=api14s, api10s=api10s, **kwargs, ) exc_time = round(timer() - ts, 2) self.add_metric( operation="download", name="*", seconds=exc_time, count=geoms.locations.shape[0], ) return geoms except Exception as e: count = len(util.ensure_list(api10s or api14s)) self.raise_execution_error( operation="download", record_count=count, e=e, extra={ "api10s": api10s, "api14s": api14s }, )
async def download( self, api14s: Union[str, List[str]] = None, api10s: Union[str, List[str]] = None, **kwargs, ) -> WellSet: kwargs = {**self.download_kwargs, **kwargs} try: ts = timer() # TODO: Add sample option wellset = await pd.DataFrame.wells.from_multiple( hole_dir=self.hole_dir, api14s=api14s, api10s=api10s, **kwargs, ) exc_time = round(timer() - ts, 2) self.add_metric( operation="download", name="*", seconds=exc_time, count=wellset.wells.shape[0], ) return wellset except Exception as e: count = len(util.ensure_list(api10s or api14s)) self.raise_execution_error( operation="download", record_count=count, e=e, extra={ "api10s": api10s, "api14s": api14s }, )
async def agg( self, funcs: Union[Function, List[Function]], filter: Union[str, TextClause, BinaryExpression] = None, ) -> Dict[str, Union[int, float]]: func_map: Dict[str, Function] = { f.name: f for f in util.ensure_list(funcs) } stmt = self.model.select(*func_map.values()) if filter is not None: if not isinstance(filter, (TextClause, BinaryExpression)): filter = text(filter) stmt = stmt.where(filter) result: db.Row async with db.Session() as session: async with session.begin(): result = (await session.execute(stmt)).one() return dict(zip(func_map, result))
async def _get( cls, ids: Union[str, List[str]], path: IHSPath, param_name: str, params: Dict = None, timeout: Optional[int] = None, concurrency: int = None, **kwargs, ) -> List[Dict[str, Any]]: responses: List[httpx.Response] = [] ids = util.ensure_list(ids) concurrency = concurrency or 50 timeout = timeout or 300 params = params or {} async with cls(**kwargs) as client: coros: List[Coroutine] = [] for id in ids: coro = client.get( path.value, params={param_name: id, **params}, timeout=timeout, ) coros.append(coro) for idx, chunk in enumerate(util.chunks(coros, concurrency)): responses += await asyncio.gather(*chunk) data: List[Dict[str, Any]] = [] for r in responses: json: Dict = r.json() # type: ignore if "data" in json.keys(): data += json["data"] return data
def get_transforms(self): if self.init_transform is None: return util.ensure_list(self.transform) return [self.transform, self.init_transform]
import sqlite3 import util if __name__ == "__main__": os.nice(19) util.setup("temp/", "") all_filepaths = ['brainSegmentation.db'] conn = sqlite3.connect(util.DB_PATH) conn.text_factory = str cursor = conn.execute('''SELECT filepath, transform, filepath_reg from Images''') for (filepath, transform, filepath_reg) in cursor: all_filepaths.extend(util.ensure_list(filepath)) all_filepaths.extend(util.ensure_list(filepath_reg)) if transform is None: continue for _transform in transform.split(","): all_filepaths.append(_transform.strip()) cursor = conn.execute('''SELECT filepath, filepath_reg from Labels''') for (filepath, filepath_reg) in cursor: all_filepaths.extend(util.ensure_list(filepath)) all_filepaths.extend(util.ensure_list(filepath_reg)) for root, dirs, files in os.walk(util.DATA_FOLDER): for filepath in files: filepath = os.path.join(root, filepath).replace(util.DATA_FOLDER, "") if filepath not in all_filepaths:
def get_config(): parser = argparse.ArgumentParser( description='Detect cheat in vjudge contest.') # basic configuration parser.add_argument(dest='files', type=str, nargs='+', help='zip files of vjudge contest submission') parser.add_argument('-c', '--config', dest='config_file', help='YAML config file path') parser.add_argument('-l', dest='language', nargs='+', default=['cc'], help='language to detect') parser.add_argument( '-a', dest='add_dir', help= 'additional files to detect cheating, put them under ${add_dir}/${problem_id}' ) # preprocess configuration parser.add_argument('-r', dest='remove_duplicate', action='store_true', help='remove duplicate submissions of a user') parser.add_argument( '-u', dest='include_userid', nargs='+', default=[], help='regex of user id to be included, include all if not specified') parser.add_argument( '-iu', dest='ignore_userid', nargs='+', default=[], help='user id to be ignored, has higher priority than include') parser.add_argument('-ip', dest='ignore_problem', nargs='+', default=[], help='problem to be ignored') # moss configuration parser.add_argument( '-id', dest='moss_userid', help='moss userid, regiester one from e-mail if you don\'t have one') parser.add_argument( '-b', dest='base_dir', help='use in moss, put base files under ${base_dir}/${problem_id}/') parser.add_argument( '-m', dest='maximal_match', type=int, default=0, help= 'use in moss, maximal appearance before identified as code in base file' ) parser.add_argument('-n', dest='report_number', type=int, default=0, help='use in moss, maximal item number in report') config = parser.parse_args() if config.config_file is not None: with open(config.config_file, "r", encoding='utf-8') as f: y = yaml.load(f) config.moss_userid = config.moss_userid or y.get("moss_userid") config.remove_duplicate = config.remove_duplicate or y.get( "remove_duplicate") config.maximal_match = config.maximal_match or y.get( "maximal_match") or 10 config.report_number = config.report_number or y.get( "report_number") or 250 config.ignore_userid.extend( util.ensure_list(y.get("ignore_userid", []))) config.language.extend(util.ensure_list(y.get("language", []))) config.include_userid.extend( util.ensure_list(y.get("include_userid", []))) if len(config.include_userid) == 0: config.include_userid.append(".+") config.include_userid = list( map(lambda x: re.compile(f"^{x}$"), config.include_userid)) return config
def test_ensure_list(self, data, expected): assert util.ensure_list(data) == expected
REMOVE = False if __name__ == "__main__": os.nice(19) util.setup("temp/") all_filepaths = ['brainSegmentation.db'] conn = sqlite3.connect(util.DB_PATH) conn.text_factory = str cursor = conn.execute( '''SELECT filepath, transform, filepath_reg from Images''') for (filepath, transform, filepath_reg) in cursor: all_filepaths.extend(util.ensure_list(filepath)) all_filepaths.extend(util.ensure_list(filepath_reg)) if transform is None: continue for _transform in transform.split(","): all_filepaths.append(_transform.strip()) cursor = conn.execute('''SELECT filepath, filepath_reg from Labels''') for (filepath, filepath_reg) in cursor: all_filepaths.extend(util.ensure_list(filepath)) all_filepaths.extend(util.ensure_list(filepath_reg)) for root, dirs, files in os.walk(util.DATA_FOLDER): for filepath in files: filepath = os.path.join(root, filepath).replace(util.DATA_FOLDER, "")