def resolve_aeb(spec):
'''
Resolve an experiment spec into the full list of points (coordinates) in AEB space.
@param {dict} spec An experiment spec.
@returns {list} aeb_list Resolved array of points in AEB space.
@example
spec = spec_util.get('base.json', 'general_inner')
aeb_list = spec_util.resolve_aeb(spec)
# => [(0, 0, 0), (0, 0, 1), (1, 1, 0), (1, 1, 1)]
'''
agent_num = len(spec['agent'])
env_num = len(spec['env'])
ae_product = ps.get(spec, 'body.product')
body_num = ps.get(spec, 'body.num')
body_num_list = body_num if ps.is_list(body_num) else [body_num] * env_num
aeb_list = []
if ae_product == 'outer':
for e in range(env_num):
sub_aeb_list = list(itertools.product(range(agent_num), [e], range(body_num_list[e])))
aeb_list.extend(sub_aeb_list)
elif ae_product == 'inner':
for a, e in zip(range(agent_num), range(env_num)):
sub_aeb_list = list(itertools.product([a], [e], range(body_num_list[e])))
aeb_list.extend(sub_aeb_list)
else: # custom AEB, body_num is a aeb_list
aeb_list = [tuple(aeb) for aeb in body_num]
aeb_list.sort()
assert is_aeb_compact(aeb_list), 'Failed check: for a, e, uniq count == len (shape), and for each a,e hash, b uniq count == b len (shape)'
return aeb_list
def plot_go(
df, y_col=None, x_col='index', y2_col=None,
title=None, y_title=None, x_title=None, x_type=None,
legend_name=None, width=500, height=350, draw=True,
save=False, filename=None,
trace_class='Scatter', trace_kwargs=None, layout_kwargs=None):
'''
Quickly plot from df using trace_class, e.g. go.Scatter
1. create_label() to auto-resolve labels
2. create_layout() with go.Layout() and update(layout_kwargs)
3. spread and create go.<trace_class>() and update(trace_kwargs)
4. Create the figure and plot accordingly
@returns figure
'''
df = df.copy()
if x_col == 'index':
df['index'] = df.index.tolist()
label = create_label(y_col, x_col, title, y_title, x_title, legend_name)
layout = create_layout(
x_type=x_type, width=width, height=height, layout_kwargs=layout_kwargs,
**ps.pick(label, ['title', 'y_title', 'x_title']))
y_col_list, x_col_list = label['y_col_list'], label['x_col_list']
if y2_col is not None:
label2 = create_label(y2_col, x_col, title, y_title, x_title, legend_name)
layout.update(dict(yaxis2=dict(
rangemode='tozero', title=label2['y_title'],
side='right', overlaying='y1', anchor='x1',
)))
y2_col_list, x_col_list = label2['y_col_list'], label2['x_col_list']
label2_legend_name_list = label2['legend_name_list']
else:
y2_col_list = []
label2_legend_name_list = []
combo_y_col_list = y_col_list + y2_col_list
combo_legend_name_list = label['legend_name_list'] + label2_legend_name_list
y_col_num, x_col_num = len(combo_y_col_list), len(x_col_list)
trace_num = max(y_col_num, x_col_num)
data = []
for idx in range(trace_num):
y_c = ps.get(combo_y_col_list, idx % y_col_num)
x_c = ps.get(x_col_list, idx % x_col_num)
df_y, df_x = ps.get(df, y_c), ps.get(df, x_c)
trace = ps.get(go, trace_class)(y=df_y, x=df_x, name=combo_legend_name_list[idx])
trace.update(trace_kwargs)
if idx >= len(y_col_list):
trace.update(dict(yaxis='y2', xaxis='x1'))
data.append(trace)
figure = go.Figure(data=data, layout=layout)
if draw:
plot(figure)
if save:
save_image(figure, filename=filename)
return figure
def check_body_spec(spec):
'''Base method to check body spec for AEB space resolution'''
ae_product = ps.get(spec, 'body.product')
body_num = ps.get(spec, 'body.num')
if ae_product == 'outer':
pass
elif ae_product == 'inner':
agent_num = len(spec['agent'])
env_num = len(spec['env'])
assert agent_num == env_num, 'Agent and Env spec length must be equal for body `inner` product. Given {agent_num}, {env_num}'
else: # custom AEB
assert ps.is_list(body_num)
def transform(self, obj, seed=None):
ret = {}
if self._transform:
obj = self._transform(obj, seed)
for column, definition in self._schema.items():
source_name = definition.get('path', column)
if isinstance(source_name, str):
source_value = obj.get(source_name)
elif isinstance(source_name, list):
source_value = get(obj, source_name)
else:
raise ValueError("Invalid path: {}".format(source_name))
if source_value is None: continue
if callable(definition['type']):
parser_func = definition['type']
elif definition['type'] in self._parser_map:
parser_func = self._parser_map[definition['type']]
else:
raise ValueError("Invalid type: {}".format(definition['type']))
try:
ret[column] = parser_func(source_value)
except (ValueError, TypeError) as err:
message = "Failed to cast {} with value {} to {}".format(
column,
source_value,
definition['type'],
)
raise ValueError(message) from err
return ret
def property_(path):
"""Creates a function that returns the value at path of a given object.
Args:
path (str|list): Path value to fetch from object.
Returns:
function: Function that returns object's path value.
Example:
>>> get_data = property_('data')
>>> get_data({'data': 1})
1
>>> get_data({}) is None
True
>>> get_first = property_(0)
>>> get_first([1, 2, 3])
1
.. versionadded:: 1.0.0
.. versionchanged:: 4.0.1
Made property accessor work with deep path strings.
"""
return lambda obj: pyd.get(obj, path)
def _nth_arg(*args):
try:
position = math.ceil(float(pos))
except ValueError:
position = 0
return pyd.get(args, position)
def deep_property(path):
"""Creates a :func:`pydash.collections.pluck` style function, which returns
the key value of a given object.
Args:
key (mixed): Key value to fetch from object.
Returns:
function: Function that returns object's key value.
Example:
>>> deep_property('a.b.c')({'a': {'b': {'c': 1}}})
1
>>> deep_property('a.1.0.b')({'a': [5, [{'b': 1}]]})
1
>>> deep_property('a.1.0.b')({}) is None
True
See Also:
- :func:`deep_property` (main definition)
- :func:`deep_prop` (alias)
.. versionadded:: 1.0.0
"""
return lambda obj: pyd.get(obj, path)
def get_by_diff_key(dic: dict, diff_key: str) -> Any:
return py_.get(dic, diff_key
.replace("root", "")
.replace("><", ".")
.replace(">", "")
.replace("<", "")
.replace("'", ""))
def __init__(self, agent_spec, agent_space, a=0):
self.agent_spec = agent_spec
self.agent_space = agent_space
self.a = a
self.spec = agent_space.spec
self.info_space = agent_space.info_space
self.name = self.agent_spec['name']
self.body_a = None
self.nanflat_body_a = None # nanflatten version of bodies
self.body_num = None
AlgorithmClass = getattr(algorithm, ps.get(self.agent_spec, 'algorithm.name'))
self.algorithm = AlgorithmClass(self)
def generate_specs(spec, const='agent'):
'''
Generate benchmark specs with compatible discrete/continuous/both types:
- take a spec
- for each in benchmark envs
- use the template env spec to update spec
- append to benchmark specs
Interchange agent and env for the reversed benchmark.
'''
if const == 'agent':
const_name = ps.get(spec, 'agent.0.algorithm.name')
variant = 'env'
else:
const_name = ps.get(spec, 'env.0.name')
variant = 'agent'
filepath = f'{spec_util.SPEC_DIR}/benchmark_{const_name}.json'
if os.path.exists(filepath):
logger.info(f'Benchmark for {const_name} exists at {filepath} already, not overwriting.')
benchmark_specs = util.read(filepath)
return benchmark_specs
logger.info(f'Generating benchmark for {const_name}')
benchmark_variants = []
benchmark_specs = {}
for dist_cont, const_names in BENCHMARK[const].items():
if const_name in const_names:
benchmark_variants.extend(BENCHMARK[variant][dist_cont])
for vary_name in benchmark_variants:
vary_spec = ENV_TEMPLATES[vary_name]
spec_name = f'{const_name}_{vary_name}'
benchmark_spec = spec.copy()
benchmark_spec['name'] = spec_name
benchmark_spec[variant] = [vary_spec]
benchmark_specs[spec_name] = benchmark_spec
util.write(benchmark_specs, filepath)
logger.info(f'Benchmark for {const_name} written to {filepath}.')
return benchmark_specs
def run(self):
num_cpus = ps.get(self.spec['meta'], 'resources.num_cpus', util.NUM_CPUS)
info_spaces = []
for _s in range(self.spec['meta']['max_session']):
self.info_space.tick('session')
info_spaces.append(deepcopy(self.info_space))
if util.get_lab_mode() == 'train' and len(info_spaces) > 1:
session_datas = util.parallelize_fn(self.init_session_and_run, info_spaces, num_cpus)
else: # dont parallelize when debugging to allow render
session_datas = [self.init_session_and_run(info_space) for info_space in info_spaces]
self.session_data_dict = {data.index[0]: data for data in session_datas}
self.data = analysis.analyze_trial(self)
self.close()
return self.data
def are_valid_github_details(ghuser='', ghrepo='', ghbranch='master'):
giturl = f"https://api.github.com/repos/{ghuser}/{ghrepo}/branches"
try:
with urlopen(giturl) as url:
branches = json.loads(url.read().decode())
except HTTPError:
return False
return pydash.collections.reduce_(
branches,
lambda isValid, branchInfo:
isValid or pydash.get(branchInfo, 'name', '') == ghbranch,
False
)
def transform(self, raw_obj: RawObj, seed: typing.Any = None) -> Obj:
obj_id = None
obj_properties = {}
for column, definition in raw_obj.schema.items():
source_name = definition.get('path', column)
if isinstance(source_name, str):
source_value = raw_obj.data.get(source_name)
elif isinstance(source_name, list):
source_value = get(raw_obj.data, source_name)
else:
raise ValueError("Invalid path: {}".format(source_name))
if source_value is None: continue
if callable(definition['type']):
parser_func = definition['type']
elif definition['type'] in self._parser_map:
parser_func = self._parser_map[definition['type']]
else:
raise ValueError("Invalid type: {}".format(definition['type']))
try:
if column == 'id':
obj_id = parser_func(source_value)
else:
obj_properties[column] = parser_func(source_value)
except (ValueError, TypeError) as err:
message = "Failed to cast {} with value {} to {}".format(
column,
source_value,
definition['type'],
)
raise ValueError(message) from err
if not obj_id:
log.warning("raw object without id", collection=raw_obj.collection, properties=raw_obj.data)
return Obj(
id=obj_id,
properties=obj_properties,
collection=raw_obj.collection,
)
def list(self, request, *args, **kwargs):
return Response(get(self.get_object(), 'extras', {}))
def saved_unsaved_descriptions(self):
unsaved_descriptions = get(self, 'cloned_descriptions', [])
if self.id:
return compact(
[*list(self.descriptions.all()), *unsaved_descriptions])
return unsaved_descriptions
def __get_last_created_locale(self):
return get(self.__names_qs(dict(), 'created_at', 'desc'), '0')
def display_locale(self):
return get(self.preferred_locale, 'locale')
def get_flagships_performance(
fields: [str] = [],
basket_type: List[BasketType] = BasketType.to_list(),
asset_class: List[AssetClass] = [AssetClass.Equity],
region: List[Region] = None,
styles: List[Union[CustomBasketStyles, ResearchBasketStyles]] = None,
start: dt.date = None,
end: dt.date = None,
) -> pd.DataFrame:
"""
Retrieve performance data for flagship baskets
:param fields: Fields to retrieve in addition to bbid, mqid, name, region, basket type, \
styles, live date, and asset class
:param basket_type: Basket type(s)
:param asset_class: Asset class (defaults to Equity)
:param region: Basket region(s)
:param styles: Basket style(s)
:param start: Date for which to retrieve pricing (defaults to previous business day)
:param end: Date for which to retrieve pricing (defaults to previous business day)
:return: pricing data for flagship baskets
**Usage**
Retrieve performance data for flagship baskets
**Examples**
Retrieve performance data for flagship Asia custom baskets
>>> from gs_quant.markets.indices_utils import *
>>>
>>> get_flagships_performance(basket_type=[BasketType.CUSTOM_BASKET], region=[Region.ASIA])
**See also**
:func:`get_flagships_with_assets` :func:`get_flagship_baskets` :func:`get_flagships_constituents`
"""
start = start or prev_business_date()
end = end or prev_business_date()
fields = list(
set(fields).union(
set([
'name', 'region', 'type', 'flagship', 'isPairBasket', 'styles',
'liveDate', 'assetClass'
])))
coverage = GsDataApi.get_coverage(
dataset_id=IndicesDatasets.GSCB_FLAGSHIP.value, fields=fields)
basket_regions = [] if region is None else [r.value for r in region]
basket_styles = [] if styles is None else [s.value for s in styles]
basket_types = [b_type.value for b_type in basket_type]
baskets_map = {}
for basket in coverage:
if get(basket, 'flagship') is False or get(basket, 'isPairBasket') is True or \
region is not None and get(basket, 'region') not in basket_regions or \
get(basket, 'type') not in basket_types or \
get(basket, 'assetClass') not in [a.value for a in asset_class] or \
styles is not None and not any(s in get(basket, 'styles', []) for s in basket_styles):
continue
baskets_map[get(basket, 'assetId')] = basket
response = GsDataApi.query_data(
query=DataQuery(where=dict(assetId=list(baskets_map.keys())),
startDate=start,
endDate=end),
dataset_id=IndicesDatasets.GSCB_FLAGSHIP.value)
performance = []
for basket in response:
basket_data = baskets_map[get(basket, 'assetId')]
basket_data.update(closePrice=get(basket, 'closePrice'))
basket_data.update(date=get(basket, 'date'))
performance.append(basket_data)
return pd.DataFrame(performance)
def _no_zone_meta_lookup(self, key):
return pydash.get(
self.dashboard_sections,
f'{key}.presModelHolder.genVizDataPresModel'
'.paneColumnsData.vizDataColumns')
def get_sub_region(self):
return get(self.get_entity(), 'subRegion')
def on_post(self, req, resp):
log_flag = False
if app_config['BOTANALYTICS_LOG_FLAG'].upper() == "ON":
log_flag, req_id, botanalytics, start_time = True, str(
uuid.uuid4()), BotAnalyticsAPI(), datetime.datetime.now()
manager = ProjectconfigsManager()
doc = req.context['doc'] or {}
serviceid = get(doc, "serviceid")
train_ner = get(doc, "train_ner", default=True)
train_ir = get(doc, "train_ir", default=True)
try:
updateStatus(doc['serviceid'], train_ner, train_ir)
train_parse.delay(doc['serviceid'], train_ner, train_ir)
predict_url = get_prediction_api_url(req)
logger.info("get prediction url is %s" % str(predict_url))
markdown = markdown_creater(serviceid, predict_url)
logger.info("markdown is %s" % str(markdown))
document = {
"$set": {
"integration_markdown": markdown.get("mark_down_ui_client")
}
}
manager.update_config_by_service_id(serviceid, document)
if log_flag:
botanalytics.log(requesttype="nertraindetails",
serviceid=doc['serviceid'],
req_id=req_id,
train_entity=train_ner,
train_intent=train_ir,
ner_req_timestamp=start_time.replace(
microsecond=0).isoformat())
except InsufficientDataError as ide:
raise ide
except AssertionError as ae:
logger.exception(ae)
raise falcon.HTTPBadRequest('Invalid Configuration', ae)
except Exception as ex:
logger.exception(ex)
description = 'Internal Server Error, Please try again later'
raise falcon.HTTPServiceUnavailable('Service Outage', description,
30)
finally:
if log_flag:
end_time = datetime.datetime.now()
total_time = relativedelta(end_time, start_time)
botanalytics.log(
requesttype="nerrequests",
serviceid=doc['serviceid'],
req_id=req_id,
action="TRAIN",
ner_req_timestamp=start_time.replace(
microsecond=0).isoformat(),
ner_req_end_timestamp=end_time.replace(
microsecond=0).isoformat(),
total_action_time=(total_time.hours * 60 * 60 * 1000 +
total_time.minutes * 60 * 1000 +
total_time.seconds * 1000) +
(total_time.microseconds / 1000))
resp.set_header('X-Powered-By', 'USTGlobal Verbis')
resp.status = falcon.HTTP_200
async def transcribe_stream(args: argparse.Namespace,
core: Voice2JsonCore) -> None:
"""Speech to text from WAV file(s)."""
from rhasspyasr import Transcription
from rhasspysilence import VoiceCommand, VoiceCommandResult
# Make sure profile has been trained
assert core.check_trained(), "Not trained"
wav_sink = None
wav_dir = None
if args.wav_sink:
wav_sink_path = Path(args.wav_sink)
if wav_sink_path.is_dir():
# Directory to write WAV files
wav_dir = wav_sink_path
else:
# Single WAV file to write
wav_sink = open(args.wav_sink, "wb")
event_sink = None
if args.event_sink:
if args.event_sink == "-":
event_sink = sys.stdout
else:
event_sink = open(args.event_sink, "w")
# Record command
recorder = core.get_command_recorder()
recorder.start()
voice_command: typing.Optional[VoiceCommand] = None
# Expecting raw 16-bit, 16Khz mono audio
audio_source = await core.make_audio_source(args.audio_source)
# Audio settings
sample_rate = int(
pydash.get(core.profile, "audio.format.sample-rate-hertz", 16000))
sample_width = (
int(pydash.get(core.profile, "audio.format.sample-width-bits", 16)) //
8)
channels = int(pydash.get(core.profile, "audio.format.channel-count", 1))
# Get speech to text transcriber for profile
transcriber = core.get_transcriber(open_transcription=args.open,
debug=args.debug)
# Run transcription in separate thread
frame_queue: "Queue[typing.Optional[bytes]]" = Queue()
def audio_stream() -> typing.Iterable[bytes]:
"""Read audio chunks from queue and yield."""
frames = frame_queue.get()
while frames:
yield frames
frames = frame_queue.get()
def transcribe_proc():
"""Transcribe live audio stream indefinitely."""
while True:
# Get result of transcription
transcribe_result = transcriber.transcribe_stream(
audio_stream(), sample_rate, sample_width, channels)
_LOGGER.debug("Transcription result: %s", transcribe_result)
transcribe_result = transcribe_result or Transcription.empty()
transcribe_dict = dataclasses.asdict(transcribe_result)
transcribe_dict["timeout"] = is_timeout
print_json(transcribe_dict)
threading.Thread(target=transcribe_proc, daemon=True).start()
# True if current voice command timed out
is_timeout = False
# Number of events for pending voice command
event_count = 0
# Number of transcriptions that have happened
num_transcriptions = 0
try:
chunk = await audio_source.read(args.chunk_size)
while chunk:
# Look for speech/silence
voice_command = recorder.process_chunk(chunk)
if event_sink:
# Print outstanding events
for event in recorder.events[event_count:]:
print_json(dataclasses.asdict(event), out_file=event_sink)
event_count = len(recorder.events)
if voice_command:
is_timeout = voice_command.result == VoiceCommandResult.FAILURE
# Force transcription
frame_queue.put(None)
# Reset
audio_data = recorder.stop()
if wav_dir:
# Write WAV to directory
wav_path = (
wav_dir /
time.strftime(args.wav_filename)).with_suffix(".wav")
wav_bytes = core.buffer_to_wav(audio_data)
wav_path.write_bytes(wav_bytes)
_LOGGER.debug("Wrote %s (%s byte(s))", wav_path,
len(wav_bytes))
elif wav_sink:
# Write to WAV file
wav_bytes = core.buffer_to_wav(audio_data)
wav_sink.write(wav_bytes)
_LOGGER.debug("Wrote %s (%s byte(s))", args.wav_sink,
len(wav_bytes))
num_transcriptions += 1
# Check exit count
if (args.exit_count is not None) and (num_transcriptions >=
args.exit_count):
_LOGGER.debug("Exit count reached")
break
recorder.start()
else:
# Add to current command
frame_queue.put(chunk)
# Next audio chunk
chunk = await audio_source.read(args.chunk_size)
finally:
transcriber.stop()
try:
await audio_source.close()
except Exception:
pass
def get_region(self) -> Optional[str]:
return get(self.get_entity(), 'region')
def train(self, train_intent):
"""
:param doc:
:param n_test_percent:
:return:
"""
manager = ProjectManager()
query = {"serviceid": self.serviceid}
config = manager.find_model(query)
if config is not None:
try:
document = {
"$set": {
"ner.status": ProjectManager.STATUS_TRAINING,
"ner.status_message":
"Entity training is in progress.",
"ner.last_trained": datetime.datetime.utcnow()
}
}
if (train_intent is True):
document = {
"$set": {
"ner.status": ProjectManager.STATUS_TRAINING,
"ner.status_message":
"Entity training is in progress.",
"ir.status": ProjectManager.STATUS_HOLD,
"ir.status_message":
"Awaiting the completion of entity training.",
"ner.last_trained": datetime.datetime.utcnow()
}
}
manager.update_config(query, document)
# starting actual training
data_manager = DatasourceManager()
self.logger.info("Starting training of service %s" %
self.serviceid)
corpus = data_manager.find_model(query)
custom_entity_model = get(config, "custom_entity_model")
entity_recognizer = self.instantiate_trainer(
custom_entity_model)
trained_utterances = entity_recognizer.train(corpus)
if entity_recognizer.get_engine(
) not in get_all_corenlp_engines():
VerbisStore().save_ner(entity_recognizer,
model_type=MODEL_TYPE_NER)
###############MINIOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO##################
# send file to minio server
VerbisStore().save_ner_minio(entity_recognizer,
model_type=MODEL_TYPE_NER)
document = {
"$set": {
"utterances": trained_utterances,
}
}
data_manager.update_datasource(query, document)
document = {
"$set": {
"ner.status": ProjectManager.STATUS_TRAINED,
"ner.status_message":
"Entity training completed successfully.",
"ner.logs.train": ""
}
}
manager.update_config(query, document)
self.logger.info(
"Completed training entity recognizer for service %s" %
self.serviceid)
except (RuntimeError, Exception) as ex:
self.logger.exception(ex, exc_info=True)
self.logger.error(traceback.format_exc())
if ex == "Cannot have number of folds n_folds=3 greater than the number of samples: 2.":
ex = "Add more utterances for entity training"
document = {
"$set": {
"ner.status": ProjectManager.STATUS_TRAINING_FAILED,
"ner.status_message": ex,
"ner.logs.train": self.logger.handlers[-1].logs
}
}
manager.update_config(query, document)
else:
description = 'Unable to find project_config with given id.' \
'Please check your request params and retry'
self.logger.error(description)
def __contains__(self, field):
return bool(_.get(self.storage, field, False))
def __getitem__(self, field):
return _.get(self.storage, field, None)
def get_sub_category(self):
return get(self.get_entity(), 'subCategory')
def get_region_code(self):
return get(self.get_entity(), 'regionCode')
output_file = open('data_modified.yaml', 'w')
# read and parse the yaml data
data = load(file.read(), Loader=Loader)
# 1. add hobbies if not exists
if 'hobbies' not in data:
data['hobbies'] = ['reading', 'developing']
# print(data)
# 2. add a friend if it is not already there
if 'friends' in data:
if 'ahmed' not in data['friends']:
data['friends'].append('ahmed')
# 3. deep nested list
# get the list if exsists, default it to empty list if it
# doesn't exist, read docs here: http://bit.ly/2SEF6HX
children_names = pydash.get(data, 'parent.child.names', [])
# the following like assumes that `parent.child.names` is either
# a list or None, otherwise you will get an error if you tried to
# append a value to none-list type.
children_names.append('lolo')
# set the newly modifed list of names
pydash.set_(data, 'parent.child.names', children_names)
# save the newly modified yaml data to different file
dump(data, output_file)
def get_sub_region_code(self):
return get(self.get_entity(), 'subRegionCode')
def get_flagships_with_assets(
identifiers: List[str],
fields: [str] = [],
basket_type: List[BasketType] = BasketType.to_list(),
asset_class: List[AssetClass] = [AssetClass.Equity],
region: List[Region] = None,
styles: List[Union[CustomBasketStyles, ResearchBasketStyles]] = None,
as_of: dt.datetime = None) -> pd.DataFrame:
"""
Retrieve a list of flagship baskets containing specified assets
:param identifiers: List of asset identifiers
:param fields: Fields to retrieve in addition to mqid, name, ticker, region, basket type, \
description, styles, live date, and asset class
:param basket_type: Basket type(s)
:param asset_class: Asset class (defaults to Equity)
:param region: Basket region(s)
:param styles: Basket style(s)
:param as_of: Datetime for which to retrieve baskets (defaults to current time)
:return: flagship baskets containing specified assets
**Usage**
Retrieve a list of flagship baskets containing specified assets
**Examples**
Retrieve a list of flagship custom baskets containing 'AAPL UW' single stock
>>> from gs_quant.markets.indices_utils import *
>>>
>>> get_flagships_with_assets(identifiers=['AAPL UW'], basket_type=[BasketType.CUSTOM_BASKET])
**See also**
:func:`get_flagship_baskets` :func:`get_flagships_performance` :func:`get_flagships_constituents`
"""
fields = list(
set(fields).union(
set([
'id', 'name', 'ticker', 'region', 'type', 'description',
'styles', 'liveDate', 'assetClass'
])))
response = GsAssetApi.resolve_assets(identifier=identifiers,
fields=['id'],
limit=1)
mqids = [get(asset, '0.id') for asset in response.values()]
query = dict(fields=fields,
type=basket_type,
asset_class=asset_class,
is_pair_basket=[False],
flagship=[True],
underlying_asset_ids=mqids)
if region is not None:
query.update(region=region)
if styles is not None:
query.update(styles=styles)
response = GsAssetApi.get_many_assets_data_scroll(**query,
as_of=as_of,
limit=2000,
scroll='1m')
return pd.DataFrame(response)
def get_bbid(self):
return get(self.get_entity(), 'xref.bbid')
def get_flagships_constituents(
fields: [str] = [],
basket_type: List[BasketType] = BasketType.to_list(),
asset_class: List[AssetClass] = [AssetClass.Equity],
region: List[Region] = None,
styles: List[Union[CustomBasketStyles, ResearchBasketStyles]] = None,
start: dt.date = None,
end: dt.date = None,
) -> pd.DataFrame:
"""
Retrieve flagship baskets constituents
:param fields: Fields to retrieve in addition to mqid, name, ticker, region, basket type, \
styles, live date, and asset class
:param basket_type: Basket type(s)
:param asset_class: Asset class (defaults to Equity)
:param region: Basket region(s)
:param styles: Basket style(s)
:param start: Start date for which to retrieve constituents (defaults to previous business day)
:param end: End date for which to retrieve constituents (defaults to previous business day)
:return: flagship baskets constituents
**Usage**
Retrieve flagship baskets constituents
**Examples**
Retrieve a list of flagship baskets constituents
>>> from gs_quant.markets.indices_utils import *
>>>
>>> get_flagships_constituents()
**See also**
:func:`get_flagships_with_assets` :func:`get_flagships_performance` :func:`get_flagship_baskets`
"""
start = start or prev_business_date()
end = end or prev_business_date()
fields = list(
set(fields).union(
set([
'id', 'name', 'ticker', 'region', 'type', 'styles', 'liveDate',
'assetClass'
])))
query = dict(fields=fields,
type=basket_type,
asset_class=asset_class,
is_pair_basket=[False],
flagship=[True])
if region is not None:
query.update(region=region)
if styles is not None:
query.update(styles=styles)
basket_data = GsAssetApi.get_many_assets_data_scroll(**query,
limit=2000,
scroll='1m')
basket_map = {get(basket, 'id'): basket for basket in basket_data}
coverage = GsDataApi.get_coverage(
dataset_id=IndicesDatasets.GSCB_FLAGSHIP.value,
fields=['type', 'bbid'],
include_history=True)
cbs, rbs = [], []
for b in coverage:
_id = get(b, 'assetId')
_type = get(b, 'type')
if _id in list(basket_map.keys()):
start_date = dt.datetime.strptime(b['historyStartDate'],
'%Y-%m-%d').date()
start_date = start_date if start < start_date else start
if _type == BasketType.CUSTOM_BASKET.value:
data = GsDataApi.query_data(
query=DataQuery(where=dict(assetId=_id),
startDate=start_date,
endDate=end),
dataset_id=IndicesDatasets.GSBASKETCONSTITUENTS.value)
basket_map[_id].update(constituents=data)
cbs.append(basket_map[_id])
elif _type == BasketType.RESEARCH_BASKET.value:
data = GsDataApi.query_data(
query=DataQuery(where=dict(assetId=_id),
startDate=start_date,
endDate=end),
dataset_id=IndicesDatasets.GIRBASKETCONSTITUENTS.value)
basket_map[_id].update(constituents=data)
rbs.append(basket_map[_id])
return pd.DataFrame(cbs + rbs)
def get_alpha2(self):
return get(self.get_entity(), 'xref.alpha2')
def __get_preferred_locale(self):
return get(
self.__names_qs(dict(locale_preferred=True), 'created_at', 'desc'),
'0')
def get_tool_attribute(self, toolName, attribute):
return pydash.get(self.config, f"tools.{toolName}.{attribute}")
def _method_of(path, *_args, **_kargs):
func = pyd.partial(pyd.get(obj, path), *args, **kargs)
return func(*_args, **_kargs)
def main():
from docopt import docopt
arguments = docopt(__doc__.format(cmd=__file__,
now=datetime.now().replace(second=0, microsecond=0).isoformat()))
time_range = int(arguments['--range'])
metrics = arguments['<metric>'] or ['CPL.avg5']
end = iso8601.parse_date(arguments['--end'], default_timezone=None)
begin = end - timedelta(hours=time_range)
reader = AtopReader(arguments['--path'], arguments['--cmd'])
with AtopParser(begin, end) as parser:
for log_file in reader.atop_log_files(begin, end):
for line in log_file:
parser.add_line(line.decode())
if not len(parser.result):
sys.stderr.write('empty result\n')
sys.exit(1)
elif arguments['metrics']:
for metric in parser.available_metric_paths:
print(metric)
elif arguments['table']:
from tabulate import tabulate
print(tabulate([[time] + [py_.get(value, metric) for metric in metrics]
for time, value in parser.result.items()],
['time'] + metrics, tablefmt="plain"))
elif arguments['json']:
from json import dumps
print(dumps({time.isoformat(): {metric: py_.get(value, metric) for metric in metrics}
for time, value in parser.result.items()}))
elif arguments['csv']:
import csv
writer = csv.writer(sys.stdout)
writer.writerow(['time'] + metrics)
for time, value in parser.result.items():
writer.writerow([time.isoformat()] + [py_.get(value, metric) for metric in metrics])
elif arguments['gnuplot']:
for metric in metrics:
width = int(arguments['--width'])
height = int(arguments['--height'])
process = subprocess.Popen(["gnuplot"], stdin=subprocess.PIPE)
process.stdin.write(b"set term dumb %d %d \n" % (width, height))
process.stdin.write(b"unset border \n")
process.stdin.write(b"unset ytics \n")
process.stdin.write(b"unset xtics \n")
process.stdin.write(b"set xtics nomirror \n")
process.stdin.write(b"unset key \n")
process.stdin.write(b"set xdata time \n")
process.stdin.write(b"set format x '%H' \n")
process.stdin.write(b"set timefmt '%Y-%m-%dT%H:%M:%S' \n")
process.stdin.write(b"set datafile sep '\t' \n")
process.stdin.write(b"plot '-' using 1:2 notitle with linespoints \n")
for time, value in parser.result.items():
process.stdin.write(b"%s\t%s\n" % (str(time.isoformat()).encode('utf-8'),
str(py_.get(value, metric)).encode('utf-8')))
process.stdin.write(b"e\n")
process.stdin.flush()
process.stdin.close()
process.wait()
elif arguments['diagram']:
import diagram
width = int(arguments['--width'])
height = int(arguments['--height'])
class DiagramOptions(object):
axis = True
batch = False
color = False
encoding = 'utf-8'
function = None # None or any of diagram.FUNCTION.keys()
legend = True
palette = None # None or any of diagram.PALETTE.keys()
reverse = False
def __init__(self, **kwargs):
self.__dict__.update(kwargs)
for metric in metrics:
engine = diagram.AxisGraph(diagram.Point((width, height)), DiagramOptions())
engine.update([py_.get(value, metric) for value in parser.result.values()])
if hasattr(sys.stdout, 'buffer'):
engine.render(sys.stdout.buffer)
else:
engine.render(sys.stdout)
def test_get(case, expected):
assert _.get(*case) == expected
class Op2RedisClient(Elasticsearch):
redis_key = _.get(settings.SYNC, 'queue.key')
@query_params('parent', 'pipeline', 'refresh', 'routing', 'timeout',
'timestamp', 'ttl', 'version', 'version_type',
'wait_for_active_shards')
def create(self, index, doc_type, id, body, params=None):
redis_client.rpush(
self.redis_key,
json_util.dumps({
'op': 'create',
'index': index,
'doc_type': doc_type,
'body': body,
'id': id,
'params': params
}))
@query_params('op_type', 'parent', 'pipeline', 'refresh', 'routing',
'timeout', 'timestamp', 'ttl', 'version', 'version_type',
'wait_for_active_shards')
def index(self, index, doc_type, body, id=None, params=None):
redis_client.rpush(
self.redis_key,
json_util.dumps({
'op': 'index',
'index': index,
'doc_type': doc_type,
'body': body,
'id': id,
'params': params
}))
@query_params('_source', '_source_exclude', '_source_include', 'fields',
'lang', 'parent', 'refresh', 'retry_on_conflict', 'routing',
'timeout', 'timestamp', 'ttl', 'version', 'version_type',
'wait_for_active_shards')
def update(self, index, doc_type, id, body=None, params=None):
redis_client.rpush(
self.redis_key,
json_util.dumps({
'op': 'update',
'index': index,
'doc_type': doc_type,
'body': body,
'id': id,
'params': params
}))
@query_params('parent', 'refresh', 'routing', 'timeout', 'version',
'version_type', 'wait_for_active_shards')
def delete(self, index, doc_type, id, params=None):
redis_client.rpush(
self.redis_key,
json_util.dumps({
'op': 'delete',
'index': index,
'doc_type': doc_type,
'id': id,
'params': params
}))
@query_params('_source', '_source_exclude', '_source_include', 'fields',
'pipeline', 'refresh', 'routing', 'timeout',
'wait_for_active_shards')
def bulk(self, body, index=None, doc_type=None, params=None):
redis_client.rpush(
self.redis_key,
json_util.dumps({
'op': 'bulk',
'index': index,
'doc_type': doc_type,
'body': body,
'params': params
}))
# 兼容Elasticsearch原生sdk返回结果
return {'errors': None, 'items': []}
@query_params('_source', '_source_exclude', '_source_include',
'allow_no_indices', 'analyze_wildcard', 'analyzer',
'conflicts', 'default_operator', 'df', 'expand_wildcards',
'from_', 'ignore_unavailable', 'lenient', 'pipeline',
'preference', 'q', 'refresh', 'request_cache',
'requests_per_second', 'routing', 'scroll', 'scroll_size',
'search_timeout', 'search_type', 'size', 'slices', 'sort',
'stats', 'terminate_after', 'timeout', 'version',
'version_type', 'wait_for_active_shards',
'wait_for_completion')
def update_by_query(self, index, doc_type=None, body=None, params=None):
redis_client.rpush(
self.redis_key,
json_util.dumps({
'op': 'update_by_query',
'index': index,
'doc_type': doc_type,
'body': body,
'params': params
}))
@query_params('_source', '_source_exclude', '_source_include',
'allow_no_indices', 'analyze_wildcard', 'analyzer',
'conflicts', 'default_operator', 'df', 'expand_wildcards',
'from_', 'ignore_unavailable', 'lenient', 'preference', 'q',
'refresh', 'request_cache', 'requests_per_second', 'routing',
'scroll', 'scroll_size', 'search_timeout', 'search_type',
'size', 'slices', 'sort', 'stats', 'terminate_after',
'timeout', 'version', 'wait_for_active_shards',
'wait_for_completion')
def delete_by_query(self, index, body, doc_type=None, params=None):
redis_client.rpush(
self.redis_key,
json_util.dumps({
'op': 'delete_by_query',
'index': index,
'doc_type': doc_type,
'body': body,
'params': params
}))
def update_with(obj, path, updater, customizer=None):
"""This method is like :func:`update` except that it accepts customizer
which is invoked to produce the objects of path. If customizer returns
``None``, path creation is handled by the method instead. The customizer is
invoked with three arguments: ``(nested_value, key, nested_object)``.
Args:
obj (list|dict): Object to modify.
path (str|list): A string or list of keys that describe the object path
to modify.
updater (function): Function that returns updated value.
customizer (function, optional): The function to customize assigned
values.
Returns:
mixed: Updated `obj`.
Warning:
`obj` is modified in place.
Example:
>>> update_with({}, '[0][1]', lambda: 'a', lambda: {})
{0: {1: 'a'}}
.. versionadded:: 4.0.0
"""
if not callable(updater):
updater = pyd.constant(updater)
if customizer is not None and not callable(customizer):
call_customizer = partial(callit, clone, customizer, argcount=1)
elif customizer:
call_customizer = partial(callit, customizer,
argcount=getargcount(customizer, maxargs=3))
else:
call_customizer = None
default_type = dict if isinstance(obj, dict) else list
tokens = to_path_tokens(path)
if not pyd.is_list(tokens): # pragma: no cover
tokens = [tokens]
last_key = pyd.last(tokens)
if isinstance(last_key, PathToken):
last_key = last_key.key
target = obj
for idx, token in enumerate(pyd.initial(tokens)):
if isinstance(token, PathToken):
key = token.key
default_factory = pyd.get(tokens,
[idx + 1, 'default_factory'],
default=default_type)
else:
key = token
default_factory = default_type
obj_val = base_get(target, key, default=None)
path_obj = None
if call_customizer:
path_obj = call_customizer(obj_val, key, target)
if path_obj is None:
path_obj = default_factory()
base_set(target, key, path_obj, allow_override=False)
try:
target = target[key]
except TypeError as exc: # pragma: no cover
try:
target = target[int(key)]
_failed = False
except Exception:
_failed = True
if _failed:
raise TypeError('Unable to update object at index {!r}. {}'
.format(key, exc))
value = base_get(target, last_key, default=None)
base_set(target, last_key, callit(updater, value))
return obj
def iso_639_1_locale(self):
return get(self.__names_qs(dict(type=ISO_639_1)), '0.name')
def config(args=None):
'''
Parameters
----------
args : argparse.Namespace, optional
Arguments as parsed by :func:`parse_args`.
See also
--------
:func:`parse_args`
Returns
-------
configobj.ConfigObj
Parsed (and potentially modified) configuration.
'''
if args is None:
args = parse_args()
config = md.config.Config(args.config)
if args.command == 'locate':
print config.filename
elif args.command == 'show':
if args.get:
data = pydash.get(config.data.dict(), args.get)
else:
data = config.data.dict()
if args.json:
# Output in JSON.
json.dump(obj=data, fp=sys.stdout, indent=4)
elif args.yaml:
# Output in YAML format.
print yaml.dump(data, default_flow_style=False),
elif isinstance(data, dict):
# Output in `ini` format.
output = io.BytesIO()
configobj.ConfigObj(data).write(output)
print output.getvalue(),
else:
print data
elif args.command == 'edit':
for action_i in ('append', 'prepend', 'set', 'remove', 'remove_key'):
if getattr(args, action_i):
action = action_i
break
if action in ('append', 'prepend', 'set', 'remove'):
# Unpack key and new value.
key, new_value = getattr(args, action)
# Look up existing value.
config_value = pydash.get(config.data, key)
if action == 'set':
# Set a key to a string value.
# Create dictionary structure containing only the specified key
# and value.
nested_value = pydash.set_({}, key, new_value)
# Merge nested value into existing configuration structure.
pydash.merge(config.data, nested_value)
else:
# Action is a list action.
if config_value is None:
# Create dictionary structure containing only empty list for
# specified key.
config_value = []
nested_value = pydash.set_({}, key, config_value)
# Merge nested value into existing configuration structure.
pydash.merge(config.data, nested_value)
elif not isinstance(config_value, list):
print >> sys.stderr, 'Value at %s is not a list.' % key
raise SystemExit(1)
if new_value in config_value:
# Remove value even if we are appending or prepending to
# avoid duplicate values.
config_value.remove(new_value)
if args.append:
config_value.append(new_value)
elif args.prepend:
config_value.insert(0, new_value)
elif action == 'remove_key':
key = getattr(args, action)
if pydash.get(config.data, key) is not None:
# Key exists.
# Split key into levels.
# Use [negative lookbehind assertion][1] to only split on
# non-escaped '.' characters.
#
# [1]: https://stackoverflow.com/a/21107911/345236
levels = re.split(r'(?<!\\)\.', key)
parents = levels[:-1]
parent = config.data
for parent_i in parents:
parent = parent[parent_i]
# Delete key from deepest parent.
del parent[levels[-1]]
if args.dry_run:
output = io.BytesIO()
config.data.write(output)
print output.getvalue(),
else:
config.save()
return config
def custom_validation_schema(self):
return get(self, 'parent.custom_validation_schema')
def display_name(self):
return get(self.preferred_locale, 'name')
def is_eligible(name):
return all(
[get(name, key) == value for key, value in filters.items()])
def get_name(self) -> Optional[str]:
return get(self.get_entity(), 'name')
def get_category(self) -> Optional[str]:
return get(self.get_entity(), 'category')
def get_country_code(self):
return get(self.get_entity(), 'xref.countryCode')
def plot(*args, **kwargs):
if util.is_jupyter():
return py.iplot(*args, **kwargs)
else:
kwargs.update({'auto_open': ps.get(kwargs, 'auto_open', False)})
return py.plot(*args, **kwargs)
