def sampling(api, line, target_amount):
"""
default sampling plugin
:param api: plugin api
:param line: data raws
:param target_amount: target amount of points
:return:
"""
# Assume timestamp, value, range is not nullable
if len(line) > target_amount and len(line) > 2:
period = api.get_abstract(
).period # timestamp is marked as the start time of a period
start_time = line[0][0]
end_time = line[-1][0]
amount = (end_time -
start_time) / period # point amount without sampling
aggr_period = utils.iceil(amount,
target_amount) / target_amount * period
start_time = utils.ifloor(line[0][0], aggr_period)
tmp = {
timestamp: []
for timestamp in range(start_time, end_time +
period, aggr_period)
}
for point in line:
tmp[utils.ifloor(point[0], aggr_period)].append(point)
line = [[
timestamp,
utils.mean(points, lambda x: x[1]),
utils.mean(points, lambda x: x[2])
] for timestamp, points in sorted(tmp.items())]
return 'default', line
def _get_bands(self, data_service, base_line, start_time, end_time):
band_service = BandService(data_service.get_id())
band_names = band_service.get_band_names()
window = end_time - start_time
# use aggr period
period = data_service.get_period()
y_axis_max = data_service.get_abstract().y_axis_max
bands, lines = [], []
for band_name, in band_names:
band_name = urllib.unquote(band_name)
band_items = band_service.get_band_items(band_name, start_time,
end_time)
band = {'name': band_name, 'bands': []}
line = {'name': band_name, 'type': 'area', 'data': []}
if len(band_items) == 0:
bands.append(band)
lines.append(line)
continue
tmp = set([])
for band_item in band_items:
for x in range(
utils.iceil(band_item.start_time, period) - period / 2,
utils.ifloor(band_item.end_time, period) + period / 2,
period):
tmp.add(x)
for timestamp in range(
utils.iceil(start_time, period) - period / 2,
utils.ifloor(end_time, period) + period / 2, period):
if timestamp in tmp:
line['data'].append([timestamp * 1000, y_axis_max])
else:
line['data'].append([timestamp * 1000, None])
lines.append(line)
band_count = band_service.get_band_count(band_name)
pre_band = band_service.get_band_item(band_name,
band_items[0].index - 1)
next_band = band_service.get_band_item(band_name,
band_items[-1].index - 1)
band_items = [
band_item.view(band_count, window) for band_item in band_items
]
for x in range(1, len(band_items) - 1):
band_items[x]['preTime'] = band_items[x -
1]['currentTime']['show']
band_items[x]['nextTime'] = band_items[
x + 1]['currentTime']['show']
if pre_band:
band_items[0]['preTime'] = pre_band.view(
band_count, window)['currentTime']['show']
if pre_band:
band_items[0]['nextTime'] = next_band.view(
band_count, window)['currentTime']['show']
band['bands'] = band_items
return bands, lines
def post(self, dataName):
"""
ref: web_api.yaml
:param dataName:
:return:
"""
data_name = utils.encode_if_unicode(dataName)
data_service = DataService(data_name)
if data_service.exists():
return self.render(msg='%s is exists' % data_name, status=422)
if len(request.files) < 1:
return self.render(msg='expect file input', status=422)
upload_file = request.files.values()[0]
try:
# parses the incoming csv into long format
# points, time_formatter = self._parse_file(upload_file) # parse data in csv
points, time_formatter = self._parse_file_pd(
upload_file) # parse data in csv
except Exception as e:
return self.render(msg=e.message, status=422)
if len(points) < 2:
return self.render(msg='at least 2 point', status=422)
timestamps = np.asarray(sorted(points.keys()))
periods = np.diff(timestamps)
period = int(np.median(periods))
start_time = utils.ifloor(timestamps.min(), period)
end_time = utils.ifloor(timestamps.max(), period) + period
# # drop those points not divisible by period
# data_raw = []
# data_raw_list = []
# in_points = 0
# for timestamp in range(start_time, end_time, period):
# if timestamp in points:
# point = points[timestamp]
# data_raw.append(
# Raw(timestamp=point[0], value=point[1], label=point[2]))
# data_raw_list.append((point[0], point[1], None))
# in_points += 1
# else:
# data_raw.append(Raw(timestamp=timestamp))
# data_raw_list.append((timestamp, None, None))
# use raw points regardless of divisibility by period
data_raw = []
data_raw_list = []
for timestamp, point in points.iteritems():
data_raw.append(
Raw(timestamp=point[0], value=point[1], label=point[2]))
data_raw_list.append((point[0], point[1], None))
logger.debug("""
start_time: {},
end_time: {},
period: {}""".format(start_time, end_time, period))
plugin = Plugin(data_service)
try:
_, (axis_min,
axis_max) = plugin('y_axis',
data_raw_list) # cal y_axis for data
except Exception as e:
logger.error(
"Error implementing 'y_axis' plugin\nError: {}".format(
e.message))
raise e
data_abstract = DataAbstract( # save abstract for data
start_time=start_time,
end_time=end_time,
y_axis_min=axis_min,
y_axis_max=axis_max,
period=period,
period_ratio=len(periods[periods == period]) * 1. / len(periods),
label_ratio=sum([1 for point in data_raw if point.label]) * 1. /
len(data_raw),
time_formatter=time_formatter.__name__)
data_service.abstract = data_abstract
try:
_, thumb = plugin('sampling', data_raw_list,
config.SAMPLE_PIXELS) # init thumb for data
except Exception as e:
logger.error("Error calling 'sampling' plugin\nError: {}".format(
e.message))
raise e
thumb = Thumb(thumb=json.dumps(
{
'msg': 'OK',
'server': request.host,
'traceId': '',
'data': {
'data': [(point[0] * 1000, point[1]) for point in thumb],
'name': 'thumb',
'type': 'line'
}
},
ensure_ascii=False))
refs = plugin('reference', data_raw_list) # init ref for data
bands = plugin('init_band', data_raw_list) # init band
data_service.set(data_abstract, data_raw, thumb, refs, bands)
return self.render(data=data_service.get_abstract().view(),
status=201,
header={'Location': '/v1/data/%s' % data_name})
def post(self, dataName):
"""
ref: web_api.yaml
:param dataName:
:return:
"""
data_name = utils.encode_if_unicode(dataName)
data_service = DataService(data_name)
if data_service.exists():
return self.render(msg='%s is exists' % data_name, status=422)
if len(request.files) < 1:
return self.render(msg='expect file input', status=422)
upload_file = request.files['file']
current_app.logger.info('Info: %s %s %s', request.files['file'], request.files, request.files['file'].filename)
try:
points, time_formatter = self._parse_file(upload_file) # parse data in csv
except Exception as e:
return self.render(msg=str(e), status=422)
if len(points) < 2:
return self.render(msg='at least 2 point', status=422)
timestamps = np.asarray(sorted(points.keys()))
periods = np.diff(timestamps)
period = int(np.median(periods))
start_time = utils.ifloor(timestamps.min(), period)
end_time = utils.ifloor(timestamps.max(), period) + period
data_raw = [] # drop those points not divisible by period
data_raw_list = []
for timestamp in range(start_time, end_time, period):
if timestamp in points:
point = points[timestamp]
data_raw.append(
Raw(timestamp=point[0], value=point[1], label=point[2]))
data_raw_list.append((point[0], point[1], None))
else:
data_raw.append(Raw(timestamp=timestamp))
data_raw_list.append((timestamp, None, None))
plugin = Plugin(data_service)
#for item in data_raw_list:
_, (axis_min, axis_max) = plugin('y_axis', data_raw_list) # cal y_axis for data
data_abstract = DataAbstract( # save abstract for data
start_time=start_time,
end_time=end_time,
y_axis_min=axis_min,
y_axis_max=axis_max,
period=period,
period_ratio=len(periods[periods == period]) * 1. / len(periods),
label_ratio=sum([1 for point in data_raw if point.label]) * 1. / len(data_raw),
time_formatter=time_formatter.__name__
)
data_service.abstract = data_abstract
_, thumb = plugin('sampling', data_raw_list, config.SAMPLE_PIXELS) # init thumb for data
thumb = Thumb(thumb=json.dumps({
'msg': 'OK',
'server': request.host,
'traceId': '',
'data': {
'data': [(point[0] * 1000, point[1]) for point in thumb],
'name': 'thumb',
'type': 'line'
}
}, ensure_ascii=False))
refs = plugin('reference', data_raw_list) # init ref for data
bands = plugin('init_band', data_raw_list) # init band
data_service.set(data_abstract, data_raw, thumb, refs, bands)
return self.render(
data=data_service.get_abstract().view(),
status=201,
header={'Location': '/v1/data/%s' % data_name}
)