def generate(self, ffrom, tto):
period_type = self.get_period_type()
ret, new_ffrom, new_tto = self._reused_data(ffrom, tto)
if new_ffrom is None and new_tto is None: # full reuse - ret == old_data
return ret
gc = gspread.authorize(self.credentials)
wks = getattr(gc.open(self.spreadsheet), self.sheet)
date_column = self.config.get("date_column", 1)
data_column = self.config.get("data_column", 2)
start_row = self.config.get("start_row", 0)
data_values = wks.col_values(data_column)
data = {}
for i, v in enumerate(wks.col_values(date_column)):
if i < start_row:
continue
if not v:
continue
data[ensure_date(v)] = parse_float(data_values[i] or "0")
return [{
"label": d.strftime("%Y-%m-%d"),
"data": data.get(d, 0)
} for d in generate_date_series(ffrom, tto, period_type)]
def generate(self, ffrom, tto):
period_type = self.get_period_type()
file_name = self.config["filename"]
date_column = self.config.get("date_column", 0)
data_column = self.config.get("data_column", 1)
filter_column = self.config.get("filter_column", -1)
filter_value = self.config.get("filter_value", "")
skip_rows = self.config.get("skip_rows", 0)
f = csv.reader(open(file_name))
if filter_column >= 0:
def filter_row(r):
return r[filter_column] == filter_value
else:
def filter_row(r):
return True
data = dict((ensure_date(row[date_column]), float(row[data_column]))
for i, row in enumerate(f)
if i >= skip_rows and row[date_column] and filter_row(row))
return [{
"label": d.strftime("%Y-%m-%d"),
"data": data.get(d, 0)
} for d in generate_date_series(ffrom, tto, period_type)]
def generate(self, ffrom, tto):
period_type = self.get_period_type()
sum_type = self.config.get("sum_type", "sum")
ret, new_ffrom, new_tto = self._reused_data(ffrom, tto)
if new_ffrom is None and new_tto is None: # full reuse - ret == old_data
return ret
query = self._get_query()
with self.db.cursor() as cursor:
for period_from in generate_date_series(new_ffrom, new_tto,
period_type):
period_to = calculate_period_to(period_from, period_type)
params = {
'ffrom': period_from,
'tto': period_to,
"period": period_type
}
params.update(self.get_params())
try:
cursor.execute(query, params)
except:
import ipdb
ipdb.set_trace()
rrow = cursor.fetchone()
if not rrow:
ret[period_from] = 0
else:
ret[period_from] = rrow["value"]
totalized_movs = sorted(ret.items(), key=lambda k_v: k_v[0])
accumulate(totalized_movs, sum_type)
if [row for row in totalized_movs if row[1] is None]:
import ipdb
ipdb.set_trace()
data = [{
"label": row[0].strftime("%Y-%m-%d"),
"data": float(row[1])
} for row in totalized_movs]
return data
def generate(self, ffrom, tto):
source = self.config["source"]
source_data = read_data(source, self.global_config)
ret = []
for month in generate_date_series(ffrom, tto, self.get_period_type()):
label = month.strftime("%Y-%m-%d")
datas = [
d["data"] for d in source_data
if d["label"][:7] == label[:7] and d["data"]
]
if datas:
data = sum(datas) / len(datas)
else:
data = 0
ret.append({"label": label, "data": data})
return ret
def generate(self, ffrom, tto):
period_type = self.get_period_type()
sum_type = self.config.get("sum_type", "sum")
ret, new_ffrom, new_tto = self._reused_data(ffrom, tto)
if new_ffrom is None and new_tto is None: # full reuse - ret == old_data
return ret
query = self._get_query()
with self.db.cursor() as cursor:
params = {
'ffrom': new_ffrom,
'tto': new_tto,
"period": period_type
}
params.update(self.get_params())
try:
cursor.execute(query, params)
except:
import ipdb
ipdb.set_trace()
raise
for row in cursor.fetchall():
ret[ensure_date(row["period"])] = row["value"]
# Fill missing dates with zeros
missing_dates = [
d
for d in generate_date_series(new_ffrom, new_tto, period_type)
if d not in ret
]
for missing in missing_dates:
ret[missing] = 0
totalized_movs = sorted(ret.items(), key=lambda k_v: k_v[0])
accumulate(totalized_movs, sum_type)
data = [{
"label": row[0].strftime("%Y-%m-%d"),
"data": float(row[1])
} for row in totalized_movs]
return data
def generate(self, ffrom, tto):
period_type = self.get_period_type()
ret, new_ffrom, new_tto = self._reused_data(ffrom, tto)
if new_ffrom is None and new_tto is None: # full reuse - ret == old_data
return ret
values = self.profile.core.query.metrics(self.metric_name).daily(
new_ffrom, days=(new_tto - new_ffrom).days + 1).values
# Agrupo por período
for i, v in enumerate(values):
date = new_ffrom + datetime.timedelta(days=i)
period_start = date_to_period(period_type, date)
ret[period_start] = v + ret.get(period_start, 0)
return [{
"label": d.strftime("%Y-%m-%d"),
"data": ret.get(d, 0)
} for d in generate_date_series(ffrom, tto, period_type)]
def _reused_data(self, ffrom, tto):
reuse = self.config.get("reuse", self.global_config.get("default_reuse", False))
period_type = self.get_period_type()
if reuse:
old_data = read_data(self.name, self.global_config)
ret = dict((parse_date(v["label"]).date(), v["data"]) for v in old_data)
missing_dates = [d for d in generate_date_series(ffrom, tto, period_type)
if not d in ret]
if not missing_dates:
return old_data, None, None
new_ffrom = missing_dates[0]
if period_type == "weekS":
delta = relativedelta(days=7)
else:
delta = relativedelta(**{"%ss" % period_type: 1})
new_tto = missing_dates[-1] + delta - relativedelta(days=1)
else:
new_ffrom = ffrom
new_tto = tto
ret = {}
return ret, new_ffrom, new_tto
def generate(self, ffrom, tto):
period_type = self.get_period_type()
url = self.base_url + self.path
params = {
"limit": self.limit,
}
for k, v in self.filters.items():
params[self.filter_format.format(k)] = v
ret, new_ffrom, new_tto = self._reused_data(ffrom, tto)
if new_ffrom is None and new_tto is None: # full reuse - ret == old_data
return ret
for period_from in generate_date_series(new_ffrom, new_tto, period_type):
period_to = calculate_period_to(period_from, period_type)
period_params = dict(params)
period_params.update({
self.filter_format.format("date_from"): period_from.strftime("%Y-%m-%d"),
self.filter_format.format("date_to"): period_to.strftime("%Y-%m-%d"),
})
resp = requests.get(url + "?{}".format(parse.urlencode(period_params)),
headers={"API-Key": self.api_key})
resp.raise_for_status()
resp = resp.json()
ret[period_from] = resp["pagination"]["total_count"]
totalized_movs = sorted(ret.items(), key=lambda k_v: k_v[0])
data = [{"label": row[0].strftime("%Y-%m-%d"), "data": float(row[1])}
for row in totalized_movs]
return data