def resolve_aeb(spec): ''' Resolve an experiment spec into the full list of points (coordinates) in AEB space. @param {dict} spec An experiment spec. @returns {list} aeb_list Resolved array of points in AEB space. @example spec = spec_util.get('base.json', 'general_inner') aeb_list = spec_util.resolve_aeb(spec) # => [(0, 0, 0), (0, 0, 1), (1, 1, 0), (1, 1, 1)] ''' agent_num = len(spec['agent']) env_num = len(spec['env']) ae_product = ps.get(spec, 'body.product') body_num = ps.get(spec, 'body.num') body_num_list = body_num if ps.is_list(body_num) else [body_num] * env_num aeb_list = [] if ae_product == 'outer': for e in range(env_num): sub_aeb_list = list(itertools.product(range(agent_num), [e], range(body_num_list[e]))) aeb_list.extend(sub_aeb_list) elif ae_product == 'inner': for a, e in zip(range(agent_num), range(env_num)): sub_aeb_list = list(itertools.product([a], [e], range(body_num_list[e]))) aeb_list.extend(sub_aeb_list) else: # custom AEB, body_num is a aeb_list aeb_list = [tuple(aeb) for aeb in body_num] aeb_list.sort() assert is_aeb_compact(aeb_list), 'Failed check: for a, e, uniq count == len (shape), and for each a,e hash, b uniq count == b len (shape)' return aeb_list
def plot_go( df, y_col=None, x_col='index', y2_col=None, title=None, y_title=None, x_title=None, x_type=None, legend_name=None, width=500, height=350, draw=True, save=False, filename=None, trace_class='Scatter', trace_kwargs=None, layout_kwargs=None): ''' Quickly plot from df using trace_class, e.g. go.Scatter 1. create_label() to auto-resolve labels 2. create_layout() with go.Layout() and update(layout_kwargs) 3. spread and create go.<trace_class>() and update(trace_kwargs) 4. Create the figure and plot accordingly @returns figure ''' df = df.copy() if x_col == 'index': df['index'] = df.index.tolist() label = create_label(y_col, x_col, title, y_title, x_title, legend_name) layout = create_layout( x_type=x_type, width=width, height=height, layout_kwargs=layout_kwargs, **ps.pick(label, ['title', 'y_title', 'x_title'])) y_col_list, x_col_list = label['y_col_list'], label['x_col_list'] if y2_col is not None: label2 = create_label(y2_col, x_col, title, y_title, x_title, legend_name) layout.update(dict(yaxis2=dict( rangemode='tozero', title=label2['y_title'], side='right', overlaying='y1', anchor='x1', ))) y2_col_list, x_col_list = label2['y_col_list'], label2['x_col_list'] label2_legend_name_list = label2['legend_name_list'] else: y2_col_list = [] label2_legend_name_list = [] combo_y_col_list = y_col_list + y2_col_list combo_legend_name_list = label['legend_name_list'] + label2_legend_name_list y_col_num, x_col_num = len(combo_y_col_list), len(x_col_list) trace_num = max(y_col_num, x_col_num) data = [] for idx in range(trace_num): y_c = ps.get(combo_y_col_list, idx % y_col_num) x_c = ps.get(x_col_list, idx % x_col_num) df_y, df_x = ps.get(df, y_c), ps.get(df, x_c) trace = ps.get(go, trace_class)(y=df_y, x=df_x, name=combo_legend_name_list[idx]) trace.update(trace_kwargs) if idx >= len(y_col_list): trace.update(dict(yaxis='y2', xaxis='x1')) data.append(trace) figure = go.Figure(data=data, layout=layout) if draw: plot(figure) if save: save_image(figure, filename=filename) return figure
def check_body_spec(spec): '''Base method to check body spec for AEB space resolution''' ae_product = ps.get(spec, 'body.product') body_num = ps.get(spec, 'body.num') if ae_product == 'outer': pass elif ae_product == 'inner': agent_num = len(spec['agent']) env_num = len(spec['env']) assert agent_num == env_num, 'Agent and Env spec length must be equal for body `inner` product. Given {agent_num}, {env_num}' else: # custom AEB assert ps.is_list(body_num)
def transform(self, obj, seed=None): ret = {} if self._transform: obj = self._transform(obj, seed) for column, definition in self._schema.items(): source_name = definition.get('path', column) if isinstance(source_name, str): source_value = obj.get(source_name) elif isinstance(source_name, list): source_value = get(obj, source_name) else: raise ValueError("Invalid path: {}".format(source_name)) if source_value is None: continue if callable(definition['type']): parser_func = definition['type'] elif definition['type'] in self._parser_map: parser_func = self._parser_map[definition['type']] else: raise ValueError("Invalid type: {}".format(definition['type'])) try: ret[column] = parser_func(source_value) except (ValueError, TypeError) as err: message = "Failed to cast {} with value {} to {}".format( column, source_value, definition['type'], ) raise ValueError(message) from err return ret
def property_(path): """Creates a function that returns the value at path of a given object. Args: path (str|list): Path value to fetch from object. Returns: function: Function that returns object's path value. Example: >>> get_data = property_('data') >>> get_data({'data': 1}) 1 >>> get_data({}) is None True >>> get_first = property_(0) >>> get_first([1, 2, 3]) 1 .. versionadded:: 1.0.0 .. versionchanged:: 4.0.1 Made property accessor work with deep path strings. """ return lambda obj: pyd.get(obj, path)
def _nth_arg(*args): try: position = math.ceil(float(pos)) except ValueError: position = 0 return pyd.get(args, position)
def deep_property(path): """Creates a :func:`pydash.collections.pluck` style function, which returns the key value of a given object. Args: key (mixed): Key value to fetch from object. Returns: function: Function that returns object's key value. Example: >>> deep_property('a.b.c')({'a': {'b': {'c': 1}}}) 1 >>> deep_property('a.1.0.b')({'a': [5, [{'b': 1}]]}) 1 >>> deep_property('a.1.0.b')({}) is None True See Also: - :func:`deep_property` (main definition) - :func:`deep_prop` (alias) .. versionadded:: 1.0.0 """ return lambda obj: pyd.get(obj, path)
def get_by_diff_key(dic: dict, diff_key: str) -> Any: return py_.get(dic, diff_key .replace("root", "") .replace("><", ".") .replace(">", "") .replace("<", "") .replace("'", ""))
def __init__(self, agent_spec, agent_space, a=0): self.agent_spec = agent_spec self.agent_space = agent_space self.a = a self.spec = agent_space.spec self.info_space = agent_space.info_space self.name = self.agent_spec['name'] self.body_a = None self.nanflat_body_a = None # nanflatten version of bodies self.body_num = None AlgorithmClass = getattr(algorithm, ps.get(self.agent_spec, 'algorithm.name')) self.algorithm = AlgorithmClass(self)
def generate_specs(spec, const='agent'): ''' Generate benchmark specs with compatible discrete/continuous/both types: - take a spec - for each in benchmark envs - use the template env spec to update spec - append to benchmark specs Interchange agent and env for the reversed benchmark. ''' if const == 'agent': const_name = ps.get(spec, 'agent.0.algorithm.name') variant = 'env' else: const_name = ps.get(spec, 'env.0.name') variant = 'agent' filepath = f'{spec_util.SPEC_DIR}/benchmark_{const_name}.json' if os.path.exists(filepath): logger.info(f'Benchmark for {const_name} exists at {filepath} already, not overwriting.') benchmark_specs = util.read(filepath) return benchmark_specs logger.info(f'Generating benchmark for {const_name}') benchmark_variants = [] benchmark_specs = {} for dist_cont, const_names in BENCHMARK[const].items(): if const_name in const_names: benchmark_variants.extend(BENCHMARK[variant][dist_cont]) for vary_name in benchmark_variants: vary_spec = ENV_TEMPLATES[vary_name] spec_name = f'{const_name}_{vary_name}' benchmark_spec = spec.copy() benchmark_spec['name'] = spec_name benchmark_spec[variant] = [vary_spec] benchmark_specs[spec_name] = benchmark_spec util.write(benchmark_specs, filepath) logger.info(f'Benchmark for {const_name} written to {filepath}.') return benchmark_specs
def run(self): num_cpus = ps.get(self.spec['meta'], 'resources.num_cpus', util.NUM_CPUS) info_spaces = [] for _s in range(self.spec['meta']['max_session']): self.info_space.tick('session') info_spaces.append(deepcopy(self.info_space)) if util.get_lab_mode() == 'train' and len(info_spaces) > 1: session_datas = util.parallelize_fn(self.init_session_and_run, info_spaces, num_cpus) else: # dont parallelize when debugging to allow render session_datas = [self.init_session_and_run(info_space) for info_space in info_spaces] self.session_data_dict = {data.index[0]: data for data in session_datas} self.data = analysis.analyze_trial(self) self.close() return self.data
def are_valid_github_details(ghuser='', ghrepo='', ghbranch='master'): giturl = f"https://api.github.com/repos/{ghuser}/{ghrepo}/branches" try: with urlopen(giturl) as url: branches = json.loads(url.read().decode()) except HTTPError: return False return pydash.collections.reduce_( branches, lambda isValid, branchInfo: isValid or pydash.get(branchInfo, 'name', '') == ghbranch, False )
def transform(self, raw_obj: RawObj, seed: typing.Any = None) -> Obj: obj_id = None obj_properties = {} for column, definition in raw_obj.schema.items(): source_name = definition.get('path', column) if isinstance(source_name, str): source_value = raw_obj.data.get(source_name) elif isinstance(source_name, list): source_value = get(raw_obj.data, source_name) else: raise ValueError("Invalid path: {}".format(source_name)) if source_value is None: continue if callable(definition['type']): parser_func = definition['type'] elif definition['type'] in self._parser_map: parser_func = self._parser_map[definition['type']] else: raise ValueError("Invalid type: {}".format(definition['type'])) try: if column == 'id': obj_id = parser_func(source_value) else: obj_properties[column] = parser_func(source_value) except (ValueError, TypeError) as err: message = "Failed to cast {} with value {} to {}".format( column, source_value, definition['type'], ) raise ValueError(message) from err if not obj_id: log.warning("raw object without id", collection=raw_obj.collection, properties=raw_obj.data) return Obj( id=obj_id, properties=obj_properties, collection=raw_obj.collection, )
def list(self, request, *args, **kwargs): return Response(get(self.get_object(), 'extras', {}))
def saved_unsaved_descriptions(self): unsaved_descriptions = get(self, 'cloned_descriptions', []) if self.id: return compact( [*list(self.descriptions.all()), *unsaved_descriptions]) return unsaved_descriptions
def __get_last_created_locale(self): return get(self.__names_qs(dict(), 'created_at', 'desc'), '0')
def display_locale(self): return get(self.preferred_locale, 'locale')
def get_flagships_performance( fields: [str] = [], basket_type: List[BasketType] = BasketType.to_list(), asset_class: List[AssetClass] = [AssetClass.Equity], region: List[Region] = None, styles: List[Union[CustomBasketStyles, ResearchBasketStyles]] = None, start: dt.date = None, end: dt.date = None, ) -> pd.DataFrame: """ Retrieve performance data for flagship baskets :param fields: Fields to retrieve in addition to bbid, mqid, name, region, basket type, \ styles, live date, and asset class :param basket_type: Basket type(s) :param asset_class: Asset class (defaults to Equity) :param region: Basket region(s) :param styles: Basket style(s) :param start: Date for which to retrieve pricing (defaults to previous business day) :param end: Date for which to retrieve pricing (defaults to previous business day) :return: pricing data for flagship baskets **Usage** Retrieve performance data for flagship baskets **Examples** Retrieve performance data for flagship Asia custom baskets >>> from gs_quant.markets.indices_utils import * >>> >>> get_flagships_performance(basket_type=[BasketType.CUSTOM_BASKET], region=[Region.ASIA]) **See also** :func:`get_flagships_with_assets` :func:`get_flagship_baskets` :func:`get_flagships_constituents` """ start = start or prev_business_date() end = end or prev_business_date() fields = list( set(fields).union( set([ 'name', 'region', 'type', 'flagship', 'isPairBasket', 'styles', 'liveDate', 'assetClass' ]))) coverage = GsDataApi.get_coverage( dataset_id=IndicesDatasets.GSCB_FLAGSHIP.value, fields=fields) basket_regions = [] if region is None else [r.value for r in region] basket_styles = [] if styles is None else [s.value for s in styles] basket_types = [b_type.value for b_type in basket_type] baskets_map = {} for basket in coverage: if get(basket, 'flagship') is False or get(basket, 'isPairBasket') is True or \ region is not None and get(basket, 'region') not in basket_regions or \ get(basket, 'type') not in basket_types or \ get(basket, 'assetClass') not in [a.value for a in asset_class] or \ styles is not None and not any(s in get(basket, 'styles', []) for s in basket_styles): continue baskets_map[get(basket, 'assetId')] = basket response = GsDataApi.query_data( query=DataQuery(where=dict(assetId=list(baskets_map.keys())), startDate=start, endDate=end), dataset_id=IndicesDatasets.GSCB_FLAGSHIP.value) performance = [] for basket in response: basket_data = baskets_map[get(basket, 'assetId')] basket_data.update(closePrice=get(basket, 'closePrice')) basket_data.update(date=get(basket, 'date')) performance.append(basket_data) return pd.DataFrame(performance)
def _no_zone_meta_lookup(self, key): return pydash.get( self.dashboard_sections, f'{key}.presModelHolder.genVizDataPresModel' '.paneColumnsData.vizDataColumns')
def get_sub_region(self): return get(self.get_entity(), 'subRegion')
def on_post(self, req, resp): log_flag = False if app_config['BOTANALYTICS_LOG_FLAG'].upper() == "ON": log_flag, req_id, botanalytics, start_time = True, str( uuid.uuid4()), BotAnalyticsAPI(), datetime.datetime.now() manager = ProjectconfigsManager() doc = req.context['doc'] or {} serviceid = get(doc, "serviceid") train_ner = get(doc, "train_ner", default=True) train_ir = get(doc, "train_ir", default=True) try: updateStatus(doc['serviceid'], train_ner, train_ir) train_parse.delay(doc['serviceid'], train_ner, train_ir) predict_url = get_prediction_api_url(req) logger.info("get prediction url is %s" % str(predict_url)) markdown = markdown_creater(serviceid, predict_url) logger.info("markdown is %s" % str(markdown)) document = { "$set": { "integration_markdown": markdown.get("mark_down_ui_client") } } manager.update_config_by_service_id(serviceid, document) if log_flag: botanalytics.log(requesttype="nertraindetails", serviceid=doc['serviceid'], req_id=req_id, train_entity=train_ner, train_intent=train_ir, ner_req_timestamp=start_time.replace( microsecond=0).isoformat()) except InsufficientDataError as ide: raise ide except AssertionError as ae: logger.exception(ae) raise falcon.HTTPBadRequest('Invalid Configuration', ae) except Exception as ex: logger.exception(ex) description = 'Internal Server Error, Please try again later' raise falcon.HTTPServiceUnavailable('Service Outage', description, 30) finally: if log_flag: end_time = datetime.datetime.now() total_time = relativedelta(end_time, start_time) botanalytics.log( requesttype="nerrequests", serviceid=doc['serviceid'], req_id=req_id, action="TRAIN", ner_req_timestamp=start_time.replace( microsecond=0).isoformat(), ner_req_end_timestamp=end_time.replace( microsecond=0).isoformat(), total_action_time=(total_time.hours * 60 * 60 * 1000 + total_time.minutes * 60 * 1000 + total_time.seconds * 1000) + (total_time.microseconds / 1000)) resp.set_header('X-Powered-By', 'USTGlobal Verbis') resp.status = falcon.HTTP_200
async def transcribe_stream(args: argparse.Namespace, core: Voice2JsonCore) -> None: """Speech to text from WAV file(s).""" from rhasspyasr import Transcription from rhasspysilence import VoiceCommand, VoiceCommandResult # Make sure profile has been trained assert core.check_trained(), "Not trained" wav_sink = None wav_dir = None if args.wav_sink: wav_sink_path = Path(args.wav_sink) if wav_sink_path.is_dir(): # Directory to write WAV files wav_dir = wav_sink_path else: # Single WAV file to write wav_sink = open(args.wav_sink, "wb") event_sink = None if args.event_sink: if args.event_sink == "-": event_sink = sys.stdout else: event_sink = open(args.event_sink, "w") # Record command recorder = core.get_command_recorder() recorder.start() voice_command: typing.Optional[VoiceCommand] = None # Expecting raw 16-bit, 16Khz mono audio audio_source = await core.make_audio_source(args.audio_source) # Audio settings sample_rate = int( pydash.get(core.profile, "audio.format.sample-rate-hertz", 16000)) sample_width = ( int(pydash.get(core.profile, "audio.format.sample-width-bits", 16)) // 8) channels = int(pydash.get(core.profile, "audio.format.channel-count", 1)) # Get speech to text transcriber for profile transcriber = core.get_transcriber(open_transcription=args.open, debug=args.debug) # Run transcription in separate thread frame_queue: "Queue[typing.Optional[bytes]]" = Queue() def audio_stream() -> typing.Iterable[bytes]: """Read audio chunks from queue and yield.""" frames = frame_queue.get() while frames: yield frames frames = frame_queue.get() def transcribe_proc(): """Transcribe live audio stream indefinitely.""" while True: # Get result of transcription transcribe_result = transcriber.transcribe_stream( audio_stream(), sample_rate, sample_width, channels) _LOGGER.debug("Transcription result: %s", transcribe_result) transcribe_result = transcribe_result or Transcription.empty() transcribe_dict = dataclasses.asdict(transcribe_result) transcribe_dict["timeout"] = is_timeout print_json(transcribe_dict) threading.Thread(target=transcribe_proc, daemon=True).start() # True if current voice command timed out is_timeout = False # Number of events for pending voice command event_count = 0 # Number of transcriptions that have happened num_transcriptions = 0 try: chunk = await audio_source.read(args.chunk_size) while chunk: # Look for speech/silence voice_command = recorder.process_chunk(chunk) if event_sink: # Print outstanding events for event in recorder.events[event_count:]: print_json(dataclasses.asdict(event), out_file=event_sink) event_count = len(recorder.events) if voice_command: is_timeout = voice_command.result == VoiceCommandResult.FAILURE # Force transcription frame_queue.put(None) # Reset audio_data = recorder.stop() if wav_dir: # Write WAV to directory wav_path = ( wav_dir / time.strftime(args.wav_filename)).with_suffix(".wav") wav_bytes = core.buffer_to_wav(audio_data) wav_path.write_bytes(wav_bytes) _LOGGER.debug("Wrote %s (%s byte(s))", wav_path, len(wav_bytes)) elif wav_sink: # Write to WAV file wav_bytes = core.buffer_to_wav(audio_data) wav_sink.write(wav_bytes) _LOGGER.debug("Wrote %s (%s byte(s))", args.wav_sink, len(wav_bytes)) num_transcriptions += 1 # Check exit count if (args.exit_count is not None) and (num_transcriptions >= args.exit_count): _LOGGER.debug("Exit count reached") break recorder.start() else: # Add to current command frame_queue.put(chunk) # Next audio chunk chunk = await audio_source.read(args.chunk_size) finally: transcriber.stop() try: await audio_source.close() except Exception: pass
def get_region(self) -> Optional[str]: return get(self.get_entity(), 'region')
def train(self, train_intent): """ :param doc: :param n_test_percent: :return: """ manager = ProjectManager() query = {"serviceid": self.serviceid} config = manager.find_model(query) if config is not None: try: document = { "$set": { "ner.status": ProjectManager.STATUS_TRAINING, "ner.status_message": "Entity training is in progress.", "ner.last_trained": datetime.datetime.utcnow() } } if (train_intent is True): document = { "$set": { "ner.status": ProjectManager.STATUS_TRAINING, "ner.status_message": "Entity training is in progress.", "ir.status": ProjectManager.STATUS_HOLD, "ir.status_message": "Awaiting the completion of entity training.", "ner.last_trained": datetime.datetime.utcnow() } } manager.update_config(query, document) # starting actual training data_manager = DatasourceManager() self.logger.info("Starting training of service %s" % self.serviceid) corpus = data_manager.find_model(query) custom_entity_model = get(config, "custom_entity_model") entity_recognizer = self.instantiate_trainer( custom_entity_model) trained_utterances = entity_recognizer.train(corpus) if entity_recognizer.get_engine( ) not in get_all_corenlp_engines(): VerbisStore().save_ner(entity_recognizer, model_type=MODEL_TYPE_NER) ###############MINIOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO################## # send file to minio server VerbisStore().save_ner_minio(entity_recognizer, model_type=MODEL_TYPE_NER) document = { "$set": { "utterances": trained_utterances, } } data_manager.update_datasource(query, document) document = { "$set": { "ner.status": ProjectManager.STATUS_TRAINED, "ner.status_message": "Entity training completed successfully.", "ner.logs.train": "" } } manager.update_config(query, document) self.logger.info( "Completed training entity recognizer for service %s" % self.serviceid) except (RuntimeError, Exception) as ex: self.logger.exception(ex, exc_info=True) self.logger.error(traceback.format_exc()) if ex == "Cannot have number of folds n_folds=3 greater than the number of samples: 2.": ex = "Add more utterances for entity training" document = { "$set": { "ner.status": ProjectManager.STATUS_TRAINING_FAILED, "ner.status_message": ex, "ner.logs.train": self.logger.handlers[-1].logs } } manager.update_config(query, document) else: description = 'Unable to find project_config with given id.' \ 'Please check your request params and retry' self.logger.error(description)
def __contains__(self, field): return bool(_.get(self.storage, field, False))
def __getitem__(self, field): return _.get(self.storage, field, None)
def get_sub_category(self): return get(self.get_entity(), 'subCategory')
def get_region_code(self): return get(self.get_entity(), 'regionCode')
output_file = open('data_modified.yaml', 'w') # read and parse the yaml data data = load(file.read(), Loader=Loader) # 1. add hobbies if not exists if 'hobbies' not in data: data['hobbies'] = ['reading', 'developing'] # print(data) # 2. add a friend if it is not already there if 'friends' in data: if 'ahmed' not in data['friends']: data['friends'].append('ahmed') # 3. deep nested list # get the list if exsists, default it to empty list if it # doesn't exist, read docs here: http://bit.ly/2SEF6HX children_names = pydash.get(data, 'parent.child.names', []) # the following like assumes that `parent.child.names` is either # a list or None, otherwise you will get an error if you tried to # append a value to none-list type. children_names.append('lolo') # set the newly modifed list of names pydash.set_(data, 'parent.child.names', children_names) # save the newly modified yaml data to different file dump(data, output_file)
def get_sub_region_code(self): return get(self.get_entity(), 'subRegionCode')
def get_flagships_with_assets( identifiers: List[str], fields: [str] = [], basket_type: List[BasketType] = BasketType.to_list(), asset_class: List[AssetClass] = [AssetClass.Equity], region: List[Region] = None, styles: List[Union[CustomBasketStyles, ResearchBasketStyles]] = None, as_of: dt.datetime = None) -> pd.DataFrame: """ Retrieve a list of flagship baskets containing specified assets :param identifiers: List of asset identifiers :param fields: Fields to retrieve in addition to mqid, name, ticker, region, basket type, \ description, styles, live date, and asset class :param basket_type: Basket type(s) :param asset_class: Asset class (defaults to Equity) :param region: Basket region(s) :param styles: Basket style(s) :param as_of: Datetime for which to retrieve baskets (defaults to current time) :return: flagship baskets containing specified assets **Usage** Retrieve a list of flagship baskets containing specified assets **Examples** Retrieve a list of flagship custom baskets containing 'AAPL UW' single stock >>> from gs_quant.markets.indices_utils import * >>> >>> get_flagships_with_assets(identifiers=['AAPL UW'], basket_type=[BasketType.CUSTOM_BASKET]) **See also** :func:`get_flagship_baskets` :func:`get_flagships_performance` :func:`get_flagships_constituents` """ fields = list( set(fields).union( set([ 'id', 'name', 'ticker', 'region', 'type', 'description', 'styles', 'liveDate', 'assetClass' ]))) response = GsAssetApi.resolve_assets(identifier=identifiers, fields=['id'], limit=1) mqids = [get(asset, '0.id') for asset in response.values()] query = dict(fields=fields, type=basket_type, asset_class=asset_class, is_pair_basket=[False], flagship=[True], underlying_asset_ids=mqids) if region is not None: query.update(region=region) if styles is not None: query.update(styles=styles) response = GsAssetApi.get_many_assets_data_scroll(**query, as_of=as_of, limit=2000, scroll='1m') return pd.DataFrame(response)
def get_bbid(self): return get(self.get_entity(), 'xref.bbid')
def get_flagships_constituents( fields: [str] = [], basket_type: List[BasketType] = BasketType.to_list(), asset_class: List[AssetClass] = [AssetClass.Equity], region: List[Region] = None, styles: List[Union[CustomBasketStyles, ResearchBasketStyles]] = None, start: dt.date = None, end: dt.date = None, ) -> pd.DataFrame: """ Retrieve flagship baskets constituents :param fields: Fields to retrieve in addition to mqid, name, ticker, region, basket type, \ styles, live date, and asset class :param basket_type: Basket type(s) :param asset_class: Asset class (defaults to Equity) :param region: Basket region(s) :param styles: Basket style(s) :param start: Start date for which to retrieve constituents (defaults to previous business day) :param end: End date for which to retrieve constituents (defaults to previous business day) :return: flagship baskets constituents **Usage** Retrieve flagship baskets constituents **Examples** Retrieve a list of flagship baskets constituents >>> from gs_quant.markets.indices_utils import * >>> >>> get_flagships_constituents() **See also** :func:`get_flagships_with_assets` :func:`get_flagships_performance` :func:`get_flagship_baskets` """ start = start or prev_business_date() end = end or prev_business_date() fields = list( set(fields).union( set([ 'id', 'name', 'ticker', 'region', 'type', 'styles', 'liveDate', 'assetClass' ]))) query = dict(fields=fields, type=basket_type, asset_class=asset_class, is_pair_basket=[False], flagship=[True]) if region is not None: query.update(region=region) if styles is not None: query.update(styles=styles) basket_data = GsAssetApi.get_many_assets_data_scroll(**query, limit=2000, scroll='1m') basket_map = {get(basket, 'id'): basket for basket in basket_data} coverage = GsDataApi.get_coverage( dataset_id=IndicesDatasets.GSCB_FLAGSHIP.value, fields=['type', 'bbid'], include_history=True) cbs, rbs = [], [] for b in coverage: _id = get(b, 'assetId') _type = get(b, 'type') if _id in list(basket_map.keys()): start_date = dt.datetime.strptime(b['historyStartDate'], '%Y-%m-%d').date() start_date = start_date if start < start_date else start if _type == BasketType.CUSTOM_BASKET.value: data = GsDataApi.query_data( query=DataQuery(where=dict(assetId=_id), startDate=start_date, endDate=end), dataset_id=IndicesDatasets.GSBASKETCONSTITUENTS.value) basket_map[_id].update(constituents=data) cbs.append(basket_map[_id]) elif _type == BasketType.RESEARCH_BASKET.value: data = GsDataApi.query_data( query=DataQuery(where=dict(assetId=_id), startDate=start_date, endDate=end), dataset_id=IndicesDatasets.GIRBASKETCONSTITUENTS.value) basket_map[_id].update(constituents=data) rbs.append(basket_map[_id]) return pd.DataFrame(cbs + rbs)
def get_alpha2(self): return get(self.get_entity(), 'xref.alpha2')
def __get_preferred_locale(self): return get( self.__names_qs(dict(locale_preferred=True), 'created_at', 'desc'), '0')
def get_tool_attribute(self, toolName, attribute): return pydash.get(self.config, f"tools.{toolName}.{attribute}")
def _method_of(path, *_args, **_kargs): func = pyd.partial(pyd.get(obj, path), *args, **kargs) return func(*_args, **_kargs)
def main(): from docopt import docopt arguments = docopt(__doc__.format(cmd=__file__, now=datetime.now().replace(second=0, microsecond=0).isoformat())) time_range = int(arguments['--range']) metrics = arguments['<metric>'] or ['CPL.avg5'] end = iso8601.parse_date(arguments['--end'], default_timezone=None) begin = end - timedelta(hours=time_range) reader = AtopReader(arguments['--path'], arguments['--cmd']) with AtopParser(begin, end) as parser: for log_file in reader.atop_log_files(begin, end): for line in log_file: parser.add_line(line.decode()) if not len(parser.result): sys.stderr.write('empty result\n') sys.exit(1) elif arguments['metrics']: for metric in parser.available_metric_paths: print(metric) elif arguments['table']: from tabulate import tabulate print(tabulate([[time] + [py_.get(value, metric) for metric in metrics] for time, value in parser.result.items()], ['time'] + metrics, tablefmt="plain")) elif arguments['json']: from json import dumps print(dumps({time.isoformat(): {metric: py_.get(value, metric) for metric in metrics} for time, value in parser.result.items()})) elif arguments['csv']: import csv writer = csv.writer(sys.stdout) writer.writerow(['time'] + metrics) for time, value in parser.result.items(): writer.writerow([time.isoformat()] + [py_.get(value, metric) for metric in metrics]) elif arguments['gnuplot']: for metric in metrics: width = int(arguments['--width']) height = int(arguments['--height']) process = subprocess.Popen(["gnuplot"], stdin=subprocess.PIPE) process.stdin.write(b"set term dumb %d %d \n" % (width, height)) process.stdin.write(b"unset border \n") process.stdin.write(b"unset ytics \n") process.stdin.write(b"unset xtics \n") process.stdin.write(b"set xtics nomirror \n") process.stdin.write(b"unset key \n") process.stdin.write(b"set xdata time \n") process.stdin.write(b"set format x '%H' \n") process.stdin.write(b"set timefmt '%Y-%m-%dT%H:%M:%S' \n") process.stdin.write(b"set datafile sep '\t' \n") process.stdin.write(b"plot '-' using 1:2 notitle with linespoints \n") for time, value in parser.result.items(): process.stdin.write(b"%s\t%s\n" % (str(time.isoformat()).encode('utf-8'), str(py_.get(value, metric)).encode('utf-8'))) process.stdin.write(b"e\n") process.stdin.flush() process.stdin.close() process.wait() elif arguments['diagram']: import diagram width = int(arguments['--width']) height = int(arguments['--height']) class DiagramOptions(object): axis = True batch = False color = False encoding = 'utf-8' function = None # None or any of diagram.FUNCTION.keys() legend = True palette = None # None or any of diagram.PALETTE.keys() reverse = False def __init__(self, **kwargs): self.__dict__.update(kwargs) for metric in metrics: engine = diagram.AxisGraph(diagram.Point((width, height)), DiagramOptions()) engine.update([py_.get(value, metric) for value in parser.result.values()]) if hasattr(sys.stdout, 'buffer'): engine.render(sys.stdout.buffer) else: engine.render(sys.stdout)
def test_get(case, expected): assert _.get(*case) == expected
class Op2RedisClient(Elasticsearch): redis_key = _.get(settings.SYNC, 'queue.key') @query_params('parent', 'pipeline', 'refresh', 'routing', 'timeout', 'timestamp', 'ttl', 'version', 'version_type', 'wait_for_active_shards') def create(self, index, doc_type, id, body, params=None): redis_client.rpush( self.redis_key, json_util.dumps({ 'op': 'create', 'index': index, 'doc_type': doc_type, 'body': body, 'id': id, 'params': params })) @query_params('op_type', 'parent', 'pipeline', 'refresh', 'routing', 'timeout', 'timestamp', 'ttl', 'version', 'version_type', 'wait_for_active_shards') def index(self, index, doc_type, body, id=None, params=None): redis_client.rpush( self.redis_key, json_util.dumps({ 'op': 'index', 'index': index, 'doc_type': doc_type, 'body': body, 'id': id, 'params': params })) @query_params('_source', '_source_exclude', '_source_include', 'fields', 'lang', 'parent', 'refresh', 'retry_on_conflict', 'routing', 'timeout', 'timestamp', 'ttl', 'version', 'version_type', 'wait_for_active_shards') def update(self, index, doc_type, id, body=None, params=None): redis_client.rpush( self.redis_key, json_util.dumps({ 'op': 'update', 'index': index, 'doc_type': doc_type, 'body': body, 'id': id, 'params': params })) @query_params('parent', 'refresh', 'routing', 'timeout', 'version', 'version_type', 'wait_for_active_shards') def delete(self, index, doc_type, id, params=None): redis_client.rpush( self.redis_key, json_util.dumps({ 'op': 'delete', 'index': index, 'doc_type': doc_type, 'id': id, 'params': params })) @query_params('_source', '_source_exclude', '_source_include', 'fields', 'pipeline', 'refresh', 'routing', 'timeout', 'wait_for_active_shards') def bulk(self, body, index=None, doc_type=None, params=None): redis_client.rpush( self.redis_key, json_util.dumps({ 'op': 'bulk', 'index': index, 'doc_type': doc_type, 'body': body, 'params': params })) # 兼容Elasticsearch原生sdk返回结果 return {'errors': None, 'items': []} @query_params('_source', '_source_exclude', '_source_include', 'allow_no_indices', 'analyze_wildcard', 'analyzer', 'conflicts', 'default_operator', 'df', 'expand_wildcards', 'from_', 'ignore_unavailable', 'lenient', 'pipeline', 'preference', 'q', 'refresh', 'request_cache', 'requests_per_second', 'routing', 'scroll', 'scroll_size', 'search_timeout', 'search_type', 'size', 'slices', 'sort', 'stats', 'terminate_after', 'timeout', 'version', 'version_type', 'wait_for_active_shards', 'wait_for_completion') def update_by_query(self, index, doc_type=None, body=None, params=None): redis_client.rpush( self.redis_key, json_util.dumps({ 'op': 'update_by_query', 'index': index, 'doc_type': doc_type, 'body': body, 'params': params })) @query_params('_source', '_source_exclude', '_source_include', 'allow_no_indices', 'analyze_wildcard', 'analyzer', 'conflicts', 'default_operator', 'df', 'expand_wildcards', 'from_', 'ignore_unavailable', 'lenient', 'preference', 'q', 'refresh', 'request_cache', 'requests_per_second', 'routing', 'scroll', 'scroll_size', 'search_timeout', 'search_type', 'size', 'slices', 'sort', 'stats', 'terminate_after', 'timeout', 'version', 'wait_for_active_shards', 'wait_for_completion') def delete_by_query(self, index, body, doc_type=None, params=None): redis_client.rpush( self.redis_key, json_util.dumps({ 'op': 'delete_by_query', 'index': index, 'doc_type': doc_type, 'body': body, 'params': params }))
def update_with(obj, path, updater, customizer=None): """This method is like :func:`update` except that it accepts customizer which is invoked to produce the objects of path. If customizer returns ``None``, path creation is handled by the method instead. The customizer is invoked with three arguments: ``(nested_value, key, nested_object)``. Args: obj (list|dict): Object to modify. path (str|list): A string or list of keys that describe the object path to modify. updater (function): Function that returns updated value. customizer (function, optional): The function to customize assigned values. Returns: mixed: Updated `obj`. Warning: `obj` is modified in place. Example: >>> update_with({}, '[0][1]', lambda: 'a', lambda: {}) {0: {1: 'a'}} .. versionadded:: 4.0.0 """ if not callable(updater): updater = pyd.constant(updater) if customizer is not None and not callable(customizer): call_customizer = partial(callit, clone, customizer, argcount=1) elif customizer: call_customizer = partial(callit, customizer, argcount=getargcount(customizer, maxargs=3)) else: call_customizer = None default_type = dict if isinstance(obj, dict) else list tokens = to_path_tokens(path) if not pyd.is_list(tokens): # pragma: no cover tokens = [tokens] last_key = pyd.last(tokens) if isinstance(last_key, PathToken): last_key = last_key.key target = obj for idx, token in enumerate(pyd.initial(tokens)): if isinstance(token, PathToken): key = token.key default_factory = pyd.get(tokens, [idx + 1, 'default_factory'], default=default_type) else: key = token default_factory = default_type obj_val = base_get(target, key, default=None) path_obj = None if call_customizer: path_obj = call_customizer(obj_val, key, target) if path_obj is None: path_obj = default_factory() base_set(target, key, path_obj, allow_override=False) try: target = target[key] except TypeError as exc: # pragma: no cover try: target = target[int(key)] _failed = False except Exception: _failed = True if _failed: raise TypeError('Unable to update object at index {!r}. {}' .format(key, exc)) value = base_get(target, last_key, default=None) base_set(target, last_key, callit(updater, value)) return obj
def iso_639_1_locale(self): return get(self.__names_qs(dict(type=ISO_639_1)), '0.name')
def config(args=None): ''' Parameters ---------- args : argparse.Namespace, optional Arguments as parsed by :func:`parse_args`. See also -------- :func:`parse_args` Returns ------- configobj.ConfigObj Parsed (and potentially modified) configuration. ''' if args is None: args = parse_args() config = md.config.Config(args.config) if args.command == 'locate': print config.filename elif args.command == 'show': if args.get: data = pydash.get(config.data.dict(), args.get) else: data = config.data.dict() if args.json: # Output in JSON. json.dump(obj=data, fp=sys.stdout, indent=4) elif args.yaml: # Output in YAML format. print yaml.dump(data, default_flow_style=False), elif isinstance(data, dict): # Output in `ini` format. output = io.BytesIO() configobj.ConfigObj(data).write(output) print output.getvalue(), else: print data elif args.command == 'edit': for action_i in ('append', 'prepend', 'set', 'remove', 'remove_key'): if getattr(args, action_i): action = action_i break if action in ('append', 'prepend', 'set', 'remove'): # Unpack key and new value. key, new_value = getattr(args, action) # Look up existing value. config_value = pydash.get(config.data, key) if action == 'set': # Set a key to a string value. # Create dictionary structure containing only the specified key # and value. nested_value = pydash.set_({}, key, new_value) # Merge nested value into existing configuration structure. pydash.merge(config.data, nested_value) else: # Action is a list action. if config_value is None: # Create dictionary structure containing only empty list for # specified key. config_value = [] nested_value = pydash.set_({}, key, config_value) # Merge nested value into existing configuration structure. pydash.merge(config.data, nested_value) elif not isinstance(config_value, list): print >> sys.stderr, 'Value at %s is not a list.' % key raise SystemExit(1) if new_value in config_value: # Remove value even if we are appending or prepending to # avoid duplicate values. config_value.remove(new_value) if args.append: config_value.append(new_value) elif args.prepend: config_value.insert(0, new_value) elif action == 'remove_key': key = getattr(args, action) if pydash.get(config.data, key) is not None: # Key exists. # Split key into levels. # Use [negative lookbehind assertion][1] to only split on # non-escaped '.' characters. # # [1]: https://stackoverflow.com/a/21107911/345236 levels = re.split(r'(?<!\\)\.', key) parents = levels[:-1] parent = config.data for parent_i in parents: parent = parent[parent_i] # Delete key from deepest parent. del parent[levels[-1]] if args.dry_run: output = io.BytesIO() config.data.write(output) print output.getvalue(), else: config.save() return config
def custom_validation_schema(self): return get(self, 'parent.custom_validation_schema')
def display_name(self): return get(self.preferred_locale, 'name')
def is_eligible(name): return all( [get(name, key) == value for key, value in filters.items()])
def get_name(self) -> Optional[str]: return get(self.get_entity(), 'name')
def get_category(self) -> Optional[str]: return get(self.get_entity(), 'category')
def get_country_code(self): return get(self.get_entity(), 'xref.countryCode')
def plot(*args, **kwargs): if util.is_jupyter(): return py.iplot(*args, **kwargs) else: kwargs.update({'auto_open': ps.get(kwargs, 'auto_open', False)}) return py.plot(*args, **kwargs)