def _output_monitors(monitors, args):
pillar = {}
pillar['datadog_monitors'] = {}
pillar['datadog_monitors']['api_key'] = args.api_key
pillar['datadog_monitors']['app_key'] = args.app_key
pillar['datadog_monitors']['manage_completely'] = False
_escape_jinja_tags(monitors)
# Overwrite the SafeDumper's default representation for type(None)
SafeDumper.add_representer(type(None), _represent_none)
monitors_dict = {}
attribute_list = ['type', 'query', 'message', 'tags', 'options']
for monitor in monitors:
name = monitor['name']
monitors_dict[name] = {}
for attribute in attribute_list:
monitors_dict[name][attribute] = monitor[attribute]
pillar['datadog_monitors']['monitors'] = monitors_dict
pdb.set_trace()
with open(args.output, 'w') as f:
yaml.safe_dump(pillar, f)
def addOrderedDictToYamlInterpreter():
_mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
def dict_representer(dumper, data):
return dumper.represent_dict(data.iteritems())
def dict_constructor(loader, node):
return OrderedDict(loader.construct_pairs(node))
SafeDumper.add_representer(OrderedDict, dict_representer)
SafeLoader.add_constructor(_mapping_tag, dict_constructor)
SafeDumper.add_representer(str, SafeRepresenter.represent_str)
def tweak_yaml():
_mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
def dict_constructor(loader, node):
return collections.OrderedDict(loader.construct_pairs(node))
yaml.add_constructor(_mapping_tag, dict_constructor)
def ordered_dict_serializer(self, data):
return self.represent_mapping('tag:yaml.org,2002:map', data.items())
SafeDumper.add_representer(collections.OrderedDict, ordered_dict_serializer)
# We want blanks for null values
SafeDumper.add_representer(
type(None),
lambda dumper, value: dumper.represent_scalar(u'tag:yaml.org,2002:null', '')
)
def tweak_yaml():
_mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
def dict_constructor(loader, node):
return collections.OrderedDict(loader.construct_pairs(node))
yaml.add_constructor(_mapping_tag, dict_constructor)
def ordered_dict_serializer(self, data):
return self.represent_mapping('tag:yaml.org,2002:map', data.items())
SafeDumper.add_representer(collections.OrderedDict,
ordered_dict_serializer)
# We want blanks for null values
SafeDumper.add_representer(
type(None), lambda dumper, value: dumper.represent_scalar(
u'tag:yaml.org,2002:null', ''))
for f in features:
dtmp = {
f: {
'type': 'continuous',
'transform': None,
'parameter': {
'normalization': None,
'boundaries': None
},
'ignore': False,
'scope': ['wide', 'deep']
}
}
featuresDict.update(dtmp)
SafeDumper.add_representer(
type(None), lambda dumper, value: dumper.represent_scalar(
u'tag:yaml.org,2002:null', ''))
with open('./feature.yaml', 'w') as output:
yaml.safe_dump(featuresDict, output, default_flow_style=False)
features = '''log_date|pid|pno|list_price|c_platform_price|discount|is_sale|create_date|sku_num|catid1|catid2|catid3|cat1_price|cat2_price|cat3_price|imp_uv_1d|click_uv_1d|ctr_uv_1d|acr_uv_1d|wr_uv_1d|imp_uv_7d|click_uv_7d|ctr_uv_7d|acr_uv_7d|wr_uv_7d|imp_uv_15d|click_uv_15d|ctr_uv_15d|acr_uv_15d|wr_uv_15d|imp_uv_30d|click_uv_30d|ctr_uv_30d|acr_uv_30d|wr_uv_30d|imp_uv_60d|click_uv_60d|ctr_uv_60d|acr_uv_60d|wr_uv_60d|imp_uv_90d|click_uv_90d|ctr_uv_90d|acr_uv_90d|wr_uv_90d|comment_cnt_1d|score_description_1d|score_quality_1d|score_size_1d|score_1d|good_score_rate_1d|imp_1d|click_1d|add_1d|add_uv_1d|wishlist_1d|wishlist_uv_1d|ctr_1d|acr_1d|wr_1d|sales_1d|in_sales_1d|orders_1d|price_1d|gmv_1d|buyers_1d|in_buyers_1d|buyer_male_1d|buyer_female_1d|buyer_neutral_1d|male_rate_1d|female_rate_1d|neutral_rate_1d|confirm_sales_1d|in_confirm_sales_1d|confirm_orders_1d|confirm_price_1d|confirm_gmv_1d|confirm_buyers_1d|in_confirm_buyers_1d|confirm_buyer_male_1d|confirm_buyer_female_1d|confirm_buyer_neutral_1d|confirm_male_rate_1d|confirm_female_rate_1d|confirm_neutral_rate_1d|refund_1d|refund_rate_1d|repurchase_rate_1d|comment_cnt_7d|score_description_7d|score_quality_7d|score_size_7d|score_7d|good_score_rate_7d|imp_7d|click_7d|add_7d|add_uv_7d|wishlist_7d|wishlist_uv_7d|ctr_7d|acr_7d|wr_7d|sales_7d|in_sales_7d|orders_7d|price_7d|gmv_7d|buyers_7d|in_buyers_7d|buyer_male_7d|buyer_female_7d|buyer_neutral_7d|male_rate_7d|female_rate_7d|neutral_rate_7d|confirm_sales_7d|in_confirm_sales_7d|confirm_orders_7d|confirm_price_7d|confirm_gmv_7d|confirm_buyers_7d|in_confirm_buyers_7d|confirm_buyer_male_7d|confirm_buyer_female_7d|confirm_buyer_neutral_7d|confirm_male_rate_7d|confirm_female_rate_7d|confirm_neutral_rate_7d|refund_7d|refund_rate_7d|repurchase_rate_7d|comment_cnt_15d|score_description_15d|score_quality_15d|score_size_15d|score_15d|good_score_rate_15d|imp_15d|click_15d|add_15d|add_uv_15d|wishlist_15d|wishlist_uv_15d|ctr_15d|acr_15d|wr_15d|sales_15d|in_sales_15d|orders_15d|price_15d|gmv_15d|buyers_15d|in_buyers_15d|buyer_male_15d|buyer_female_15d|buyer_neutral_15d|male_rate_15d|female_rate_15d|neutral_rate_15d|confirm_sales_15d|in_confirm_sales_15d|confirm_orders_15d|confirm_price_15d|confirm_gmv_15d|confirm_buyers_15d|in_confirm_buyers_15d|confirm_buyer_male_15d|confirm_buyer_female_15d|confirm_buyer_neutral_15d|confirm_male_rate_15d|confirm_female_rate_15d|confirm_neutral_rate_15d|refund_15d|refund_rate_15d|repurchase_rate_15d|comment_cnt_30d|score_description_30d|score_quality_30d|score_size_30d|score_30d|good_score_rate_30d|imp_30d|click_30d|add_30d|add_uv_30d|wishlist_30d|wishlist_uv_30d|ctr_30d|acr_30d|wr_30d|sales_30d|in_sales_30d|orders_30d|price_30d|gmv_30d|buyers_30d|in_buyers_30d|buyer_male_30d|buyer_female_30d|buyer_neutral_30d|male_rate_30d|female_rate_30d|neutral_rate_30d|confirm_sales_30d|in_confirm_sales_30d|confirm_orders_30d|confirm_price_30d|confirm_gmv_30d|confirm_buyers_30d|in_confirm_buyers_30d|confirm_buyer_male_30d|confirm_buyer_female_30d|confirm_buyer_neutral_30d|confirm_male_rate_30d|confirm_female_rate_30d|confirm_neutral_rate_30d|refund_30d|refund_rate_30d|repurchase_rate_30d|comment_cnt_60d|score_description_60d|score_quality_60d|score_size_60d|score_60d|good_score_rate_60d|imp_60d|click_60d|add_60d|add_uv_60d|wishlist_60d|wishlist_uv_60d|ctr_60d|acr_60d|wr_60d|sales_60d|in_sales_60d|orders_60d|price_60d|gmv_60d|buyers_60d|in_buyers_60d|buyer_male_60d|buyer_female_60d|buyer_neutral_60d|male_rate_60d|female_rate_60d|neutral_rate_60d|confirm_sales_60d|in_confirm_sales_60d|confirm_orders_60d|confirm_price_60d|confirm_gmv_60d|confirm_buyers_60d|in_confirm_buyers_60d|confirm_buyer_male_60d|confirm_buyer_female_60d|confirm_buyer_neutral_60d|confirm_male_rate_60d|confirm_female_rate_60d|confirm_neutral_rate_60d|refund_60d|refund_rate_60d|repurchase_rate_60d|comment_cnt_90d|score_description_90d|score_quality_90d|score_size_90d|score_90d|good_score_rate_90d|imp_90d|click_90d|add_90d|add_uv_90d|wishlist_90d|wishlist_uv_90d|ctr_90d|acr_90d|wr_90d|sales_90d|in_sales_90d|orders_90d|price_90d|gmv_90d|buyers_90d|in_buyers_90d|buyer_male_90d|buyer_female_90d|buyer_neutral_90d|male_rate_90d|female_rate_90d|neutral_rate_90d|confirm_sales_90d|in_confirm_sales_90d|confirm_orders_90d|confirm_price_90d|confirm_gmv_90d|confirm_buyers_90d|in_confirm_buyers_90d|confirm_buyer_male_90d|confirm_buyer_female_90d|confirm_buyer_neutral_90d|confirm_male_rate_90d|confirm_female_rate_90d|confirm_neutral_rate_90d|refund_90d|refund_rate_90d|repurchase_rate_90d|comment_cnt|score_description|score_quality|score_size|score|good_score_rate|imp|click|add|add_uv|wishlist|wishlist_uv|ctr|acr|wr|sales|in_sales|orders|price|gmv|buyers|in_buyers|buyer_male|buyer_female|buyer_neutral|male_rate|female_rate|neutral_rate|confirm_sales|in_confirm_sales|confirm_orders|confirm_price|confirm_gmv|confirm_buyers|in_confirm_buyers|confirm_buyer_male|confirm_buyer_female|confirm_buyer_neutral|confirm_male_rate|confirm_female_rate|confirm_neutral_rate|refund|refund_rate|repurchase_rate|page_imp_1d|page_imp_uv_1d|page_imp_7d|page_imp_uv_7d|page_imp_15d|page_imp_uv_15d|page_imp_30d|page_imp_uv_30d|page_imp_60d|page_imp_uv_60d|page_imp_90d|page_imp_uv_90d|page_imp|page_imp_uv|season|gender|illegal_tags|shipping_sales_1d|shipping_orders_1d|shipping_sales_7d|shipping_orders_7d|shipping_sales_15d|shipping_orders_15d|shipping_sales_30d|shipping_orders_30d|shipping_sales_60d|shipping_orders_60d|shipping_sales_90d|shipping_orders_90d|shipping_sales|shipping_orders|confirm_price_unit_india_var_1d|confirm_price_unit_india_var_7d|confirm_price_unit_india_var_15d|confirm_price_unit_india_var_30d|confirm_price_unit_india_var_60d|confirm_price_unit_india_var_90d|confirm_price_unit_india_var|confirm_price_unit_india_avg_1d|confirm_price_unit_india_avg_7d|confirm_price_unit_india_avg_15d|confirm_price_unit_india_avg_30d|confirm_price_unit_india_avg_60d|confirm_price_unit_india_avg_90d|confirm_price_unit_india_avg'''
features = features.split('|')
featuresSchema = {i + 1: '' + v + '' for i, v in enumerate(features)}
with open('./schema.yaml', 'w') as output:
yaml.safe_dump(featuresSchema, output, default_flow_style=False)
l = []
for cat in cat3['RECORDS']:
l.append(cat['cate_three_id'])
:type src: DataPoint
"""
if self[self.TIMESTAMP] != src[self.TIMESTAMP]:
msg = "Cannot merge different timestamps (%s and %s)"
raise TaurusInternalException(msg % (self[self.TIMESTAMP], src[self.TIMESTAMP]))
self[DataPoint.SUBRESULTS].append(src)
self.__merge_kpis(src[self.CURRENT], self[self.CURRENT], src[DataPoint.SOURCE_ID])
self.__merge_kpis(src[self.CUMULATIVE], self[self.CUMULATIVE], src[DataPoint.SOURCE_ID])
if do_recalculate:
self.recalculate()
SafeDumper.add_representer(KPISet, SafeRepresenter.represent_dict)
SafeDumper.add_representer(DataPoint, SafeRepresenter.represent_dict)
class ResultsProvider(object):
"""
:type listeners: list[AggregatorListener]
"""
def __init__(self):
super(ResultsProvider, self).__init__()
self.cumulative = {}
self.track_percentiles = [0.0, 50.0, 90.0, 95.0, 99.0, 99.9, 100.0]
self.listeners = []
self.buffer_len = 2
self.min_buffer_len = 2
r"\1" + (" " * self.tab_replacement_spaces) + r"\3", line)
if replaced != line:
line = replaced
if self.warn_on_tab_replacement:
self.log.warning(
"Replaced leading tabs in file %s, line %s", fname,
num)
self.log.warning("Line content is: %s", replaced.strip())
self.log.warning(
"Please remember that YAML spec does not allow using tabs for indentation"
)
res += line
return res
SafeDumper.add_representer(Configuration, SafeRepresenter.represent_dict)
SafeDumper.add_representer(BetterDict, SafeRepresenter.represent_dict)
if PY2:
SafeDumper.add_representer(text_type, SafeRepresenter.represent_unicode)
SafeDumper.add_representer(str, str_representer)
if PY2:
# dirty hack from http://stackoverflow.com/questions/1447287/format-floats-with-standard-json-module
encoder.FLOAT_REPR = lambda o: format(o, '.3g')
else:
pass # TODO: how to implement it?
def replace_in_config(config, samples, substitutes, log=None):
def file_replacer(value, key, container):
if value in samples:
def tuple_representer(dumper: SafeRepresenter,
data: Watermark) -> SequenceNode:
return dumper.represent_list(list(data))
def watermark_representer(dumper: SafeRepresenter,
data: Watermark) -> SequenceNode:
return dumper.represent_list(list(data))
def bytes_representer(dumper: SafeRepresenter, data: bytes) -> ScalarNode:
return dumper.represent_str(data.decode("UTF-8"))
SafeDumper.add_representer(Tuple, tuple_representer)
SafeDumper.add_representer(bytes, bytes_representer)
SafeDumper.add_representer(Watermark, watermark_representer)
class BytesEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, bytes):
return obj.decode("UTF-8")
# Let the base class default method raise the TypeError
return json.JSONEncoder.default(self, obj)
def format_output(output: Any, output_format: str) -> str:
if output_format == "yaml":
return yaml.dump(output,
:type src: DataPoint
"""
if self[self.TIMESTAMP] != src[self.TIMESTAMP]:
msg = "Cannot merge different timestamps (%s and %s)"
raise TaurusInternalException(msg % (self[self.TIMESTAMP], src[self.TIMESTAMP]))
self[DataPoint.SUBRESULTS].append(src)
self.__merge_kpis(src[self.CURRENT], self[self.CURRENT], src[DataPoint.SOURCE_ID])
self.__merge_kpis(src[self.CUMULATIVE], self[self.CUMULATIVE], src[DataPoint.SOURCE_ID])
if do_recalculate:
self.recalculate()
SafeDumper.add_representer(KPISet, SafeRepresenter.represent_dict)
SafeDumper.add_representer(DataPoint, SafeRepresenter.represent_dict)
class ResultsProvider(object):
"""
:type listeners: list[AggregatorListener]
"""
def __init__(self):
super(ResultsProvider, self).__init__()
self.cumulative = {}
self.track_percentiles = [0.0, 50.0, 90.0, 95.0, 99.0, 99.9, 100.0]
self.listeners = []
self.buffer_len = 2
self.min_buffer_len = 2
for num, line in enumerate(lines):
replaced = has_tab_indents.sub(
r"\1" + (" " * self.tab_replacement_spaces) + r"\3", line)
if replaced != line:
line = replaced
if self.warn_on_tab_replacement:
self.log.warning(
"Replaced leading tabs in file %s, line %s", fname,
num)
self.log.warning("Line content is: %s", replaced.strip())
self.log.warning(
"Please remember that YAML spec does not allow using tabs for indentation"
)
res += line
return res
SafeDumper.add_representer(Configuration, SafeRepresenter.represent_dict)
SafeDumper.add_representer(BetterDict, SafeRepresenter.represent_dict)
SafeDumper.add_representer(str, str_representer)
def replace_in_config(config, samples, substitutes, log=None):
def file_replacer(value, key, container):
if value in samples:
container[key] = substitutes[samples.index(value)]
if container[key] != value and log:
log.debug("Replaced %s with %s", value, container[key])
BetterDict.traverse(config, file_replacer)
def addNobodyKnewResultToYamlInterpreter():
SafeDumper.add_representer(
NobodyKnewResult,
lambda dumper, y: dumper.represent_scalar("!nobody", ""))
SafeLoader.add_constructor("!nobody", lambda x, y: NobodyKnewResult())
for item_key, item_value in mapping:
node_key = self.represent_data(item_key)
node_value = self.represent_data(item_value)
if not (isinstance(node_key, ScalarNode) and not node_key.style):
best_style = False
if not (isinstance(node_value, ScalarNode) and not node_value.style):
best_style = False
value.append((node_key, node_value))
if flow_style is None:
if self.default_flow_style is not None:
node.flow_style = self.default_flow_style
else:
node.flow_style = best_style
return node
SafeDumper.add_representer(decimal.Decimal, SafeDumper.represent_decimal)
SafeDumper.add_representer(OrderedDict, representer.SafeRepresenter.represent_dict)
SafeDumper.add_representer(types.GeneratorType, representer.SafeRepresenter.represent_list)
@hug.format.content_type('text/yaml')
def yaml(content, **kwargs):
"""YAML (Yet Another Markup Language - A superset of JSON)"""
if hasattr(content, 'read'):
return content
if isinstance(content, tuple) and getattr(content, '_fields', None):
content = {field: getattr(content, field) for field in content._fields}
return dump(content, default_flow_style=False, Dumper=SafeDumper).encode('utf8')
def _replace_tabs(self, lines, fname):
has_tab_indents = re.compile("^( *)(\t+)( *\S*)")
res = ""
for num, line in enumerate(lines):
replaced = has_tab_indents.sub(r"\1" + (" " * self.tab_replacement_spaces) + r"\3", line)
if replaced != line:
line = replaced
if self.warn_on_tab_replacement:
self.log.warning("Replaced leading tabs in file %s, line %s", fname, num)
self.log.warning("Line content is: %s", replaced.strip())
self.log.warning("Please remember that YAML spec does not allow using tabs for indentation")
res += line
return res
SafeDumper.add_representer(Configuration, SafeRepresenter.represent_dict)
SafeDumper.add_representer(BetterDict, SafeRepresenter.represent_dict)
if PY2:
SafeDumper.add_representer(text_type, SafeRepresenter.represent_unicode)
SafeDumper.add_representer(str, str_representer)
if PY2:
# dirty hack from http://stackoverflow.com/questions/1447287/format-floats-with-standard-json-module
encoder.FLOAT_REPR = lambda o: format(o, '.3g')
else:
pass # TODO: how to implement it?
class EngineModule(object):
"""
Base class for any BZT engine module
node_value = self.represent_data(item_value)
if not (isinstance(node_key, ScalarNode) and not node_key.style):
best_style = False
if not (isinstance(node_value, ScalarNode)
and not node_value.style):
best_style = False
value.append((node_key, node_value))
if flow_style is None:
if self.default_flow_style is not None:
node.flow_style = self.default_flow_style
else:
node.flow_style = best_style
return node
SafeDumper.add_representer(decimal.Decimal, SafeDumper.represent_decimal)
SafeDumper.add_representer(OrderedDict,
representer.SafeRepresenter.represent_dict)
SafeDumper.add_representer(types.GeneratorType,
representer.SafeRepresenter.represent_list)
@hug.format.content_type('text/yaml')
def yaml(content, **kwargs):
"""YAML (Yet Another Markup Language - A superset of JSON)"""
if hasattr(content, 'read'):
return content
if isinstance(content, tuple) and getattr(content, '_fields', None):
content = {field: getattr(content, field) for field in content._fields}
