def _output_monitors(monitors, args):
pillar = {}
pillar['datadog_monitors'] = {}
pillar['datadog_monitors']['api_key'] = args.api_key
pillar['datadog_monitors']['app_key'] = args.app_key
pillar['datadog_monitors']['manage_completely'] = False
_escape_jinja_tags(monitors)
# Overwrite the SafeDumper's default representation for type(None)
SafeDumper.add_representer(type(None), _represent_none)
monitors_dict = {}
attribute_list = ['type', 'query', 'message', 'tags', 'options']
for monitor in monitors:
name = monitor['name']
monitors_dict[name] = {}
for attribute in attribute_list:
monitors_dict[name][attribute] = monitor[attribute]
pillar['datadog_monitors']['monitors'] = monitors_dict
pdb.set_trace()
with open(args.output, 'w') as f:
yaml.safe_dump(pillar, f)
def addOrderedDictToYamlInterpreter():
_mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
def dict_representer(dumper, data):
return dumper.represent_dict(data.iteritems())
def dict_constructor(loader, node):
return OrderedDict(loader.construct_pairs(node))
SafeDumper.add_representer(OrderedDict, dict_representer)
SafeLoader.add_constructor(_mapping_tag, dict_constructor)
SafeDumper.add_representer(str, SafeRepresenter.represent_str)
def to_yaml(cls, dumper: yaml.SafeDumper, data) -> yaml.nodes.MappingNode:
mapping = {
"id": data.id,
"number": data.number,
"nested": data._nested,
}
return dumper.represent_mapping(cls.yaml_tag, mapping)
def Example2_representer(dumper: yaml.SafeDumper,
obj: Example) -> yaml.nodes.MappingNode:
return dumper.represent_mapping('!Example', {
"id": obj.id,
"number": obj.number,
"field1": obj.field1,
})
def output_math_representer(dumper: yaml.SafeDumper,
event: MathOutputEvent) -> yaml.nodes.Node:
'''
Dump out a representation of a MathOutputEvent.
'''
return dumper.represent_mapping('!effect.math',
ordered_yield_math(event))
def tweak_yaml():
_mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
def dict_constructor(loader, node):
return collections.OrderedDict(loader.construct_pairs(node))
yaml.add_constructor(_mapping_tag, dict_constructor)
def ordered_dict_serializer(self, data):
return self.represent_mapping('tag:yaml.org,2002:map', data.items())
SafeDumper.add_representer(collections.OrderedDict, ordered_dict_serializer)
# We want blanks for null values
SafeDumper.add_representer(
type(None),
lambda dumper, value: dumper.represent_scalar(u'tag:yaml.org,2002:null', '')
)
def tweak_yaml():
_mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
def dict_constructor(loader, node):
return collections.OrderedDict(loader.construct_pairs(node))
yaml.add_constructor(_mapping_tag, dict_constructor)
def ordered_dict_serializer(self, data):
return self.represent_mapping('tag:yaml.org,2002:map', data.items())
SafeDumper.add_representer(collections.OrderedDict,
ordered_dict_serializer)
# We want blanks for null values
SafeDumper.add_representer(
type(None), lambda dumper, value: dumper.represent_scalar(
u'tag:yaml.org,2002:null', ''))
def regex_representer(dumper: yaml.SafeDumper,
regex: re.Pattern) -> yaml.nodes.Node:
'''
Returns a string for the regex pattern.
'''
if isinstance(regex, re.Pattern):
regex = regex.pattern
return dumper.represent_scalar('!regex', regex)
def _repr_str(dumper: SafeDumper, data: str) -> ScalarNode:
if len(data.splitlines()) > 1:
style = "|"
elif display_width(data, tabsize=_TAB) > _WIDTH:
style = ">"
else:
style = ""
node = dumper.represent_scalar("tag:yaml.org,2002:str", data, style=style)
return node
def repr_str(dumper: SafeDumper, data: str) -> ScalarNode:
if len(data.splitlines()) > 1:
style = "|"
elif len(data) > break_pt:
style = ">"
else:
style = ""
node: ScalarNode = dumper.represent_scalar(
"tag:yaml.org,2002:str", data, style=style
)
return node
def _stats_config_representer(
dumper: yaml.SafeDumper,
stats_config: StatsStorageConfig) -> yaml.nodes.ScalarNode:
"""Returns a scalar representer that instructs PyYAML how to serialize a StatsStorageConfig instance
to an "arg string" in the format of "<STORAGE_TYPE>:<RUNNING_ENVIRONMENT>:<TAG>:<PATH_OR_BUCKET>".
Args:
dumper (yaml.SafeDumper): PyYAML's default SafeDumper instance
stats_config (StatsStorageConfig): An instance of StatsStorageConfig to serialize
Returns:
yaml.nodes.ScalarNode: A scalar YAML node representing a StatsStorageConfig instance
"""
return dumper.represent_scalar('!StatsConfig', stats_config.to_arg_str())
def tuple_representer(dumper: SafeRepresenter,
data: Watermark) -> SequenceNode:
return dumper.represent_list(list(data))
def watermark_representer(dumper: SafeRepresenter,
data: Watermark) -> SequenceNode:
return dumper.represent_list(list(data))
def bytes_representer(dumper: SafeRepresenter, data: bytes) -> ScalarNode:
return dumper.represent_str(data.decode("UTF-8"))
SafeDumper.add_representer(Tuple, tuple_representer)
SafeDumper.add_representer(bytes, bytes_representer)
SafeDumper.add_representer(Watermark, watermark_representer)
class BytesEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, bytes):
return obj.decode("UTF-8")
# Let the base class default method raise the TypeError
return json.JSONEncoder.default(self, obj)
def format_output(output: Any, output_format: str) -> str:
if output_format == "yaml":
return yaml.dump(output,
:type src: DataPoint
"""
if self[self.TIMESTAMP] != src[self.TIMESTAMP]:
msg = "Cannot merge different timestamps (%s and %s)"
raise TaurusInternalException(msg % (self[self.TIMESTAMP], src[self.TIMESTAMP]))
self[DataPoint.SUBRESULTS].append(src)
self.__merge_kpis(src[self.CURRENT], self[self.CURRENT], src[DataPoint.SOURCE_ID])
self.__merge_kpis(src[self.CUMULATIVE], self[self.CUMULATIVE], src[DataPoint.SOURCE_ID])
if do_recalculate:
self.recalculate()
SafeDumper.add_representer(KPISet, SafeRepresenter.represent_dict)
SafeDumper.add_representer(DataPoint, SafeRepresenter.represent_dict)
class ResultsProvider(object):
"""
:type listeners: list[AggregatorListener]
"""
def __init__(self):
super(ResultsProvider, self).__init__()
self.cumulative = {}
self.track_percentiles = [0.0, 50.0, 90.0, 95.0, 99.0, 99.9, 100.0]
self.listeners = []
self.buffer_len = 2
self.min_buffer_len = 2
for num, line in enumerate(lines):
replaced = has_tab_indents.sub(
r"\1" + (" " * self.tab_replacement_spaces) + r"\3", line)
if replaced != line:
line = replaced
if self.warn_on_tab_replacement:
self.log.warning(
"Replaced leading tabs in file %s, line %s", fname,
num)
self.log.warning("Line content is: %s", replaced.strip())
self.log.warning(
"Please remember that YAML spec does not allow using tabs for indentation"
)
res += line
return res
SafeDumper.add_representer(Configuration, SafeRepresenter.represent_dict)
SafeDumper.add_representer(BetterDict, SafeRepresenter.represent_dict)
SafeDumper.add_representer(str, str_representer)
def replace_in_config(config, samples, substitutes, log=None):
def file_replacer(value, key, container):
if value in samples:
container[key] = substitutes[samples.index(value)]
if container[key] != value and log:
log.debug("Replaced %s with %s", value, container[key])
BetterDict.traverse(config, file_replacer)
def _replace_tabs(self, lines, fname):
has_tab_indents = re.compile("^( *)(\t+)( *\S*)")
res = ""
for num, line in enumerate(lines):
replaced = has_tab_indents.sub(r"\1" + (" " * self.tab_replacement_spaces) + r"\3", line)
if replaced != line:
line = replaced
if self.warn_on_tab_replacement:
self.log.warning("Replaced leading tabs in file %s, line %s", fname, num)
self.log.warning("Line content is: %s", replaced.strip())
self.log.warning("Please remember that YAML spec does not allow using tabs for indentation")
res += line
return res
SafeDumper.add_representer(Configuration, SafeRepresenter.represent_dict)
SafeDumper.add_representer(BetterDict, SafeRepresenter.represent_dict)
if PY2:
SafeDumper.add_representer(text_type, SafeRepresenter.represent_unicode)
SafeDumper.add_representer(str, str_representer)
if PY2:
# dirty hack from http://stackoverflow.com/questions/1447287/format-floats-with-standard-json-module
encoder.FLOAT_REPR = lambda o: format(o, '.3g')
else:
pass # TODO: how to implement it?
class EngineModule(object):
"""
Base class for any BZT engine module
def _str_enum_representer(dumper: yaml.SafeDumper, data: Any):
"""Represent a str-Enum as a string."""
return dumper.represent_str(data.value)
def time_representer(dumper: SafeDumper, data: time):
return dumper.represent_str(data.strftime("%H:%M:%S"))
def to_yaml(cls, dumper: yaml.SafeDumper, data) -> yaml.nodes.MappingNode:
mapping = {
"id": data.id,
"val": data.value,
}
return dumper.represent_mapping(cls.yaml_tag, mapping)
for f in features:
dtmp = {
f: {
'type': 'continuous',
'transform': None,
'parameter': {
'normalization': None,
'boundaries': None
},
'ignore': False,
'scope': ['wide', 'deep']
}
}
featuresDict.update(dtmp)
SafeDumper.add_representer(
type(None), lambda dumper, value: dumper.represent_scalar(
u'tag:yaml.org,2002:null', ''))
with open('./feature.yaml', 'w') as output:
yaml.safe_dump(featuresDict, output, default_flow_style=False)
features = '''log_date|pid|pno|list_price|c_platform_price|discount|is_sale|create_date|sku_num|catid1|catid2|catid3|cat1_price|cat2_price|cat3_price|imp_uv_1d|click_uv_1d|ctr_uv_1d|acr_uv_1d|wr_uv_1d|imp_uv_7d|click_uv_7d|ctr_uv_7d|acr_uv_7d|wr_uv_7d|imp_uv_15d|click_uv_15d|ctr_uv_15d|acr_uv_15d|wr_uv_15d|imp_uv_30d|click_uv_30d|ctr_uv_30d|acr_uv_30d|wr_uv_30d|imp_uv_60d|click_uv_60d|ctr_uv_60d|acr_uv_60d|wr_uv_60d|imp_uv_90d|click_uv_90d|ctr_uv_90d|acr_uv_90d|wr_uv_90d|comment_cnt_1d|score_description_1d|score_quality_1d|score_size_1d|score_1d|good_score_rate_1d|imp_1d|click_1d|add_1d|add_uv_1d|wishlist_1d|wishlist_uv_1d|ctr_1d|acr_1d|wr_1d|sales_1d|in_sales_1d|orders_1d|price_1d|gmv_1d|buyers_1d|in_buyers_1d|buyer_male_1d|buyer_female_1d|buyer_neutral_1d|male_rate_1d|female_rate_1d|neutral_rate_1d|confirm_sales_1d|in_confirm_sales_1d|confirm_orders_1d|confirm_price_1d|confirm_gmv_1d|confirm_buyers_1d|in_confirm_buyers_1d|confirm_buyer_male_1d|confirm_buyer_female_1d|confirm_buyer_neutral_1d|confirm_male_rate_1d|confirm_female_rate_1d|confirm_neutral_rate_1d|refund_1d|refund_rate_1d|repurchase_rate_1d|comment_cnt_7d|score_description_7d|score_quality_7d|score_size_7d|score_7d|good_score_rate_7d|imp_7d|click_7d|add_7d|add_uv_7d|wishlist_7d|wishlist_uv_7d|ctr_7d|acr_7d|wr_7d|sales_7d|in_sales_7d|orders_7d|price_7d|gmv_7d|buyers_7d|in_buyers_7d|buyer_male_7d|buyer_female_7d|buyer_neutral_7d|male_rate_7d|female_rate_7d|neutral_rate_7d|confirm_sales_7d|in_confirm_sales_7d|confirm_orders_7d|confirm_price_7d|confirm_gmv_7d|confirm_buyers_7d|in_confirm_buyers_7d|confirm_buyer_male_7d|confirm_buyer_female_7d|confirm_buyer_neutral_7d|confirm_male_rate_7d|confirm_female_rate_7d|confirm_neutral_rate_7d|refund_7d|refund_rate_7d|repurchase_rate_7d|comment_cnt_15d|score_description_15d|score_quality_15d|score_size_15d|score_15d|good_score_rate_15d|imp_15d|click_15d|add_15d|add_uv_15d|wishlist_15d|wishlist_uv_15d|ctr_15d|acr_15d|wr_15d|sales_15d|in_sales_15d|orders_15d|price_15d|gmv_15d|buyers_15d|in_buyers_15d|buyer_male_15d|buyer_female_15d|buyer_neutral_15d|male_rate_15d|female_rate_15d|neutral_rate_15d|confirm_sales_15d|in_confirm_sales_15d|confirm_orders_15d|confirm_price_15d|confirm_gmv_15d|confirm_buyers_15d|in_confirm_buyers_15d|confirm_buyer_male_15d|confirm_buyer_female_15d|confirm_buyer_neutral_15d|confirm_male_rate_15d|confirm_female_rate_15d|confirm_neutral_rate_15d|refund_15d|refund_rate_15d|repurchase_rate_15d|comment_cnt_30d|score_description_30d|score_quality_30d|score_size_30d|score_30d|good_score_rate_30d|imp_30d|click_30d|add_30d|add_uv_30d|wishlist_30d|wishlist_uv_30d|ctr_30d|acr_30d|wr_30d|sales_30d|in_sales_30d|orders_30d|price_30d|gmv_30d|buyers_30d|in_buyers_30d|buyer_male_30d|buyer_female_30d|buyer_neutral_30d|male_rate_30d|female_rate_30d|neutral_rate_30d|confirm_sales_30d|in_confirm_sales_30d|confirm_orders_30d|confirm_price_30d|confirm_gmv_30d|confirm_buyers_30d|in_confirm_buyers_30d|confirm_buyer_male_30d|confirm_buyer_female_30d|confirm_buyer_neutral_30d|confirm_male_rate_30d|confirm_female_rate_30d|confirm_neutral_rate_30d|refund_30d|refund_rate_30d|repurchase_rate_30d|comment_cnt_60d|score_description_60d|score_quality_60d|score_size_60d|score_60d|good_score_rate_60d|imp_60d|click_60d|add_60d|add_uv_60d|wishlist_60d|wishlist_uv_60d|ctr_60d|acr_60d|wr_60d|sales_60d|in_sales_60d|orders_60d|price_60d|gmv_60d|buyers_60d|in_buyers_60d|buyer_male_60d|buyer_female_60d|buyer_neutral_60d|male_rate_60d|female_rate_60d|neutral_rate_60d|confirm_sales_60d|in_confirm_sales_60d|confirm_orders_60d|confirm_price_60d|confirm_gmv_60d|confirm_buyers_60d|in_confirm_buyers_60d|confirm_buyer_male_60d|confirm_buyer_female_60d|confirm_buyer_neutral_60d|confirm_male_rate_60d|confirm_female_rate_60d|confirm_neutral_rate_60d|refund_60d|refund_rate_60d|repurchase_rate_60d|comment_cnt_90d|score_description_90d|score_quality_90d|score_size_90d|score_90d|good_score_rate_90d|imp_90d|click_90d|add_90d|add_uv_90d|wishlist_90d|wishlist_uv_90d|ctr_90d|acr_90d|wr_90d|sales_90d|in_sales_90d|orders_90d|price_90d|gmv_90d|buyers_90d|in_buyers_90d|buyer_male_90d|buyer_female_90d|buyer_neutral_90d|male_rate_90d|female_rate_90d|neutral_rate_90d|confirm_sales_90d|in_confirm_sales_90d|confirm_orders_90d|confirm_price_90d|confirm_gmv_90d|confirm_buyers_90d|in_confirm_buyers_90d|confirm_buyer_male_90d|confirm_buyer_female_90d|confirm_buyer_neutral_90d|confirm_male_rate_90d|confirm_female_rate_90d|confirm_neutral_rate_90d|refund_90d|refund_rate_90d|repurchase_rate_90d|comment_cnt|score_description|score_quality|score_size|score|good_score_rate|imp|click|add|add_uv|wishlist|wishlist_uv|ctr|acr|wr|sales|in_sales|orders|price|gmv|buyers|in_buyers|buyer_male|buyer_female|buyer_neutral|male_rate|female_rate|neutral_rate|confirm_sales|in_confirm_sales|confirm_orders|confirm_price|confirm_gmv|confirm_buyers|in_confirm_buyers|confirm_buyer_male|confirm_buyer_female|confirm_buyer_neutral|confirm_male_rate|confirm_female_rate|confirm_neutral_rate|refund|refund_rate|repurchase_rate|page_imp_1d|page_imp_uv_1d|page_imp_7d|page_imp_uv_7d|page_imp_15d|page_imp_uv_15d|page_imp_30d|page_imp_uv_30d|page_imp_60d|page_imp_uv_60d|page_imp_90d|page_imp_uv_90d|page_imp|page_imp_uv|season|gender|illegal_tags|shipping_sales_1d|shipping_orders_1d|shipping_sales_7d|shipping_orders_7d|shipping_sales_15d|shipping_orders_15d|shipping_sales_30d|shipping_orders_30d|shipping_sales_60d|shipping_orders_60d|shipping_sales_90d|shipping_orders_90d|shipping_sales|shipping_orders|confirm_price_unit_india_var_1d|confirm_price_unit_india_var_7d|confirm_price_unit_india_var_15d|confirm_price_unit_india_var_30d|confirm_price_unit_india_var_60d|confirm_price_unit_india_var_90d|confirm_price_unit_india_var|confirm_price_unit_india_avg_1d|confirm_price_unit_india_avg_7d|confirm_price_unit_india_avg_15d|confirm_price_unit_india_avg_30d|confirm_price_unit_india_avg_60d|confirm_price_unit_india_avg_90d|confirm_price_unit_india_avg'''
features = features.split('|')
featuresSchema = {i + 1: '' + v + '' for i, v in enumerate(features)}
with open('./schema.yaml', 'w') as output:
yaml.safe_dump(featuresSchema, output, default_flow_style=False)
l = []
for cat in cat3['RECORDS']:
l.append(cat['cate_three_id'])
def enum_representer(dumper: SafeDumper, data: ClientConfigEnum):
return dumper.represent_str(str(data))
def command_shortcut_representer(dumper: SafeDumper,
data: CommandShortcutModel):
return dumper.represent_dict(data.__dict__)
def path_representer(dumper: SafeDumper, data: Path):
return dumper.represent_str(str(data))
def addNobodyKnewResultToYamlInterpreter():
SafeDumper.add_representer(
NobodyKnewResult,
lambda dumper, y: dumper.represent_scalar("!nobody", ""))
SafeLoader.add_constructor("!nobody", lambda x, y: NobodyKnewResult())
r"\1" + (" " * self.tab_replacement_spaces) + r"\3", line)
if replaced != line:
line = replaced
if self.warn_on_tab_replacement:
self.log.warning(
"Replaced leading tabs in file %s, line %s", fname,
num)
self.log.warning("Line content is: %s", replaced.strip())
self.log.warning(
"Please remember that YAML spec does not allow using tabs for indentation"
)
res += line
return res
SafeDumper.add_representer(Configuration, SafeRepresenter.represent_dict)
SafeDumper.add_representer(BetterDict, SafeRepresenter.represent_dict)
if PY2:
SafeDumper.add_representer(text_type, SafeRepresenter.represent_unicode)
SafeDumper.add_representer(str, str_representer)
if PY2:
# dirty hack from http://stackoverflow.com/questions/1447287/format-floats-with-standard-json-module
encoder.FLOAT_REPR = lambda o: format(o, '.3g')
else:
pass # TODO: how to implement it?
def replace_in_config(config, samples, substitutes, log=None):
def file_replacer(value, key, container):
if value in samples:
node_value = self.represent_data(item_value)
if not (isinstance(node_key, ScalarNode) and not node_key.style):
best_style = False
if not (isinstance(node_value, ScalarNode)
and not node_value.style):
best_style = False
value.append((node_key, node_value))
if flow_style is None:
if self.default_flow_style is not None:
node.flow_style = self.default_flow_style
else:
node.flow_style = best_style
return node
SafeDumper.add_representer(decimal.Decimal, SafeDumper.represent_decimal)
SafeDumper.add_representer(OrderedDict,
representer.SafeRepresenter.represent_dict)
SafeDumper.add_representer(types.GeneratorType,
representer.SafeRepresenter.represent_list)
@hug.format.content_type('text/yaml')
def yaml(content, **kwargs):
"""YAML (Yet Another Markup Language - A superset of JSON)"""
if hasattr(content, 'read'):
return content
if isinstance(content, tuple) and getattr(content, '_fields', None):
content = {field: getattr(content, field) for field in content._fields}
:type src: DataPoint
"""
if self[self.TIMESTAMP] != src[self.TIMESTAMP]:
msg = "Cannot merge different timestamps (%s and %s)"
raise TaurusInternalException(msg % (self[self.TIMESTAMP], src[self.TIMESTAMP]))
self[DataPoint.SUBRESULTS].append(src)
self.__merge_kpis(src[self.CURRENT], self[self.CURRENT], src[DataPoint.SOURCE_ID])
self.__merge_kpis(src[self.CUMULATIVE], self[self.CUMULATIVE], src[DataPoint.SOURCE_ID])
if do_recalculate:
self.recalculate()
SafeDumper.add_representer(KPISet, SafeRepresenter.represent_dict)
SafeDumper.add_representer(DataPoint, SafeRepresenter.represent_dict)
class ResultsProvider(object):
"""
:type listeners: list[AggregatorListener]
"""
def __init__(self):
super(ResultsProvider, self).__init__()
self.cumulative = {}
self.track_percentiles = [0.0, 50.0, 90.0, 95.0, 99.0, 99.9, 100.0]
self.listeners = []
self.buffer_len = 2
self.min_buffer_len = 2
def _dict_representer(dumper: yaml.SafeDumper, data: Any):
"""Represent the object as a dict created of (key, value) pairs."""
return dumper.represent_dict(iter(data))
def datetime_representer(dumper: SafeDumper, data: datetime):
return dumper.represent_datetime(data)
for item_key, item_value in mapping:
node_key = self.represent_data(item_key)
node_value = self.represent_data(item_value)
if not (isinstance(node_key, ScalarNode) and not node_key.style):
best_style = False
if not (isinstance(node_value, ScalarNode) and not node_value.style):
best_style = False
value.append((node_key, node_value))
if flow_style is None:
if self.default_flow_style is not None:
node.flow_style = self.default_flow_style
else:
node.flow_style = best_style
return node
SafeDumper.add_representer(decimal.Decimal, SafeDumper.represent_decimal)
SafeDumper.add_representer(OrderedDict, representer.SafeRepresenter.represent_dict)
SafeDumper.add_representer(types.GeneratorType, representer.SafeRepresenter.represent_list)
@hug.format.content_type('text/yaml')
def yaml(content, **kwargs):
"""YAML (Yet Another Markup Language - A superset of JSON)"""
if hasattr(content, 'read'):
return content
if isinstance(content, tuple) and getattr(content, '_fields', None):
content = {field: getattr(content, field) for field in content._fields}
return dump(content, default_flow_style=False, Dumper=SafeDumper).encode('utf8')
def _repr_seq(dumper: SafeDumper, data: Sequence[Any]) -> SequenceNode:
node = dumper.represent_sequence("tag:yaml.org,2002:seq", data, flow_style=True)
return node
def _dict_representer(dumper: yaml.SafeDumper, data):
return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items())
def fix_newline_yaml_shenanigans(dumper: yaml.SafeDumper, data: str) -> str:
if '\n' in data:
return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|')
return dumper.original_represent_str(data)
def decimal_representer(dumper: SafeDumper, data: Decimal):
return dumper.represent_float(float(data))