def handle(self, **options):
# Make sure the debug flag is not set!
# When it is set, the django database wrapper collects a list of all sql
# statements executed and their timings. This consumes plenty of memory
# and cpu time.
tmp_debug = settings.DEBUG
settings.DEBUG = False
# Pick up the options
start = options["start"]
end = options["end"]
weekstart = int(options["weekstart"])
database = options["database"]
if database not in settings.DATABASES:
raise CommandError("No database settings known for '%s'" % database)
if options["user"]:
try:
user = User.objects.all().using(database).get(username=options["user"])
except:
raise CommandError("User '%s' not found" % options["user"])
else:
user = None
now = datetime.now()
task = None
try:
# Initialize the task
if options["task"]:
if options["task"] > 0:
try:
task = (
Task.objects.all().using(database).get(pk=options["task"])
)
except Task.DoesNotExist:
raise CommandError("Task identifier not found")
if (
task.started
or task.finished
or task.status != "Waiting"
or task.name not in ("frepple_createbuckets", "createbuckets")
):
raise CommandError("Invalid task identifier")
task.status = "0%"
task.started = now
else:
task = Task(
name="createbuckets",
submitted=now,
started=now,
status="0%",
user=user,
arguments="--start=%s --end=%s --weekstart=%s"
% (start, end, weekstart),
)
if task:
task.processid = os.getpid()
task.save(using=database)
# Validate the date arguments
try:
curdate = datetime.strptime(start, "%Y-%m-%d")
enddate = datetime.strptime(end, "%Y-%m-%d")
except Exception as e:
raise CommandError("Date is not matching format YYYY-MM-DD")
with transaction.atomic(using=database, savepoint=False):
# Delete previous contents
with connections[database].cursor() as cursor:
cursor.execute(
"delete from common_bucketdetail where bucket_id in ('year','quarter','month','week','day')"
)
cursor.execute(
"delete from common_bucket where name in ('year','quarter','month','week','day')"
)
# Create buckets
y = Bucket(name="year", description="Yearly time buckets", level=1)
q = Bucket(
name="quarter", description="Quarterly time buckets", level=2
)
m = Bucket(name="month", description="Monthly time buckets", level=3)
w = Bucket(name="week", description="Weeky time buckets", level=4)
d = Bucket(name="day", description="Daily time buckets", level=5)
y.save(using=database)
q.save(using=database)
m.save(using=database)
w.save(using=database)
d.save(using=database)
# Loop over all days in the chosen horizon
prev_year = None
prev_quarter = None
prev_month = None
prev_week = None
while curdate < enddate:
month = int(
curdate.strftime("%m")
) # an integer in the range 1 - 12
quarter = (month - 1) // 3 + 1 # an integer in the range 1 - 4
year = int(curdate.strftime("%Y"))
dayofweek = int(
curdate.strftime("%w")
) # day of the week, 0 = sunday, 1 = monday, ...
year_start = datetime(year, 1, 1)
year_end = datetime(year + 1, 1, 1)
week_start = curdate - timedelta(
(dayofweek + 6) % 7 + 1 - weekstart
)
week_end = curdate - timedelta((dayofweek + 6) % 7 - 6 - weekstart)
# Create buckets
if year != prev_year:
prev_year = year
BucketDetail(
bucket=y,
name=self.formatDate(curdate, options["format_year"]),
startdate=year_start,
enddate=year_end,
).save(using=database)
if quarter != prev_quarter:
prev_quarter = quarter
BucketDetail(
bucket=q,
name=self.formatDate(curdate, options["format_quarter"]),
startdate=date(year, quarter * 3 - 2, 1),
enddate=date(
year + quarter // 4,
quarter * 3 + 1 - 12 * (quarter // 4),
1,
),
).save(using=database)
if month != prev_month:
prev_month = month
BucketDetail(
bucket=m,
name=self.formatDate(curdate, options["format_month"]),
startdate=date(year, month, 1),
enddate=date(
year + month // 12, month + 1 - 12 * (month // 12), 1
),
).save(using=database)
if week_start != prev_week:
prev_week = week_start
# we need to avoid weeks 00
# we will therefore take the name of the week starting the monday
# included in that week
BucketDetail(
bucket=w,
name=self.formatDate(
week_start
+ timedelta(days=(7 - week_start.weekday()) % 7),
options["format_week"],
),
startdate=week_start,
enddate=week_end,
).save(using=database)
BucketDetail(
bucket=d,
name=self.formatDate(curdate.date(), options["format_day"]),
startdate=curdate,
enddate=curdate + timedelta(1),
).save(using=database)
# Next date
curdate = curdate + timedelta(1)
# Log success
if task:
task.status = "Done"
task.finished = datetime.now()
except Exception as e:
if task:
task.status = "Failed"
task.message = "%s" % e
task.finished = datetime.now()
raise e
finally:
if task:
task.processid = None
task.save(using=database)
settings.DEBUG = tmp_debug
def handle(self, **options):
# Make sure the debug flag is not set!
# When it is set, the django database wrapper collects a list of all sql
# statements executed and their timings. This consumes plenty of memory
# and cpu time.
tmp_debug = settings.DEBUG
settings.DEBUG = False
# Pick up the options
if 'start' in options:
start = options['start'] or '2011-1-1'
else:
start = '2011-1-1'
if 'end' in options:
end = options['end'] or '2019-1-1'
else:
end = '2019-1-1'
if 'weekstart' in options:
weekstart = int(options['weekstart'])
if weekstart < 0 or weekstart > 6:
raise CommandError("Invalid weekstart %s" % weekstart)
else:
weekstart = 1
if 'database' in options:
database = options['database'] or DEFAULT_DB_ALIAS
else:
database = DEFAULT_DB_ALIAS
if database not in settings.DATABASES:
raise CommandError("No database settings known for '%s'" % database )
if 'user' in options and options['user']:
try:
user = User.objects.all().using(database).get(username=options['user'])
except:
raise CommandError("User '%s' not found" % options['user'] )
else:
user = None
now = datetime.now()
task = None
try:
# Initialize the task
if 'task' in options and options['task']:
try:
task = Task.objects.all().using(database).get(pk=options['task'])
except:
raise CommandError("Task identifier not found")
if task.started or task.finished or task.status != "Waiting" or task.name != 'generate buckets':
raise CommandError("Invalid task identifier")
task.status = '0%'
task.started = now
else:
task = Task(name='generate buckets', submitted=now, started=now, status='0%', user=user, arguments="--start=%s --end=%s --weekstart=%s" % (start, end, weekstart))
task.save(using=database)
# Validate the date arguments
try:
curdate = datetime.strptime(start, '%Y-%m-%d')
enddate = datetime.strptime(end, '%Y-%m-%d')
except Exception as e:
raise CommandError("Date is not matching format YYYY-MM-DD")
with transaction.atomic(using=database, savepoint=False):
# Delete previous contents
connections[database].cursor().execute(
"delete from common_bucketdetail where bucket_id in ('year','quarter','month','week','day')"
)
connections[database].cursor().execute(
"delete from common_bucket where name in ('year','quarter','month','week','day')"
)
# Create buckets
y = Bucket(name='year', description='Yearly time buckets', level=1)
q = Bucket(name='quarter', description='Quarterly time buckets', level=2)
m = Bucket(name='month', description='Monthly time buckets', level=3)
w = Bucket(name='week', description='Weeky time buckets', level=4)
d = Bucket(name='day', description='Daily time buckets', level=5)
y.save(using=database)
q.save(using=database)
m.save(using=database)
w.save(using=database)
d.save(using=database)
# Loop over all days in the chosen horizon
prev_year = None
prev_quarter = None
prev_month = None
prev_week = None
while curdate < enddate:
month = int(curdate.strftime("%m")) # an integer in the range 1 - 12
quarter = (month - 1) // 3 + 1 # an integer in the range 1 - 4
year = int(curdate.strftime("%Y"))
dayofweek = int(curdate.strftime("%w")) # day of the week, 0 = sunday, 1 = monday, ...
year_start = datetime(year, 1, 1)
year_end = datetime(year + 1, 1, 1)
week_start = curdate - timedelta((dayofweek + 6) % 7 + 1 - weekstart)
week_end = curdate - timedelta((dayofweek + 6) % 7 - 6 - weekstart)
if week_start < year_start:
week_start = year_start
if week_end > year_end:
week_end = year_end
# Create buckets
if year != prev_year:
prev_year = year
BucketDetail(
bucket=y,
name=str(year),
startdate=year_start,
enddate=year_end
).save(using=database)
if quarter != prev_quarter:
prev_quarter = quarter
BucketDetail(
bucket=q,
name="%02d Q%s" % (year - 2000, quarter),
startdate=date(year, quarter * 3 - 2, 1),
enddate=date(year + quarter // 4, quarter * 3 + 1 - 12 * (quarter // 4), 1)
).save(using=database)
if month != prev_month:
prev_month = month
BucketDetail(
bucket=m,
name=curdate.strftime("%b %y"),
startdate=date(year, month, 1),
enddate=date(year + month // 12, month + 1 - 12 * (month // 12), 1),
).save(using=database)
if week_start != prev_week:
prev_week = week_start
BucketDetail(
bucket=w,
name=curdate.strftime("%y W%W"),
startdate=week_start,
enddate=week_end,
).save(using=database)
BucketDetail(
bucket=d,
name=str(curdate.date()),
startdate=curdate,
enddate=curdate + timedelta(1),
).save(using=database)
# Next date
curdate = curdate + timedelta(1)
# Log success
task.status = 'Done'
task.finished = datetime.now()
except Exception as e:
if task:
task.status = 'Failed'
task.message = '%s' % e
task.finished = datetime.now()
raise e
finally:
if task:
task.save(using=database)
settings.DEBUG = tmp_debug
def updateTelescope(min_day_horizon=10, min_week_horizon=40, min_month_horizon=730, database=DEFAULT_DB_ALIAS):
'''
Update for the telescopic horizon.
The first argument specifies the minimum number of daily buckets. Additional
daily buckets will be appended till we come to a monday. At that date weekly
buckets are starting.
The second argument specifies the minimum horizon with weeks before the
monthly buckets. The last weekly bucket can be a partial one: starting on
monday and ending on the first day of the next calendar month.
'''
# Make sure the debug flag is not set!
# When it is set, the django database wrapper collects a list of all sql
# statements executed and their timings. This consumes plenty of memory
# and cpu time.
tmp_debug = settings.DEBUG
settings.DEBUG = False
transaction.enter_transaction_management(using=database)
try:
# Delete previous contents
connections[database].cursor().execute(
"delete from common_bucketdetail where bucket_id = 'telescope'"
)
# Create bucket
try:
b = Bucket.objects.using(database).get(name='telescope')
except Bucket.DoesNotExist:
b = Bucket(name='telescope', description='Time buckets with decreasing granularity')
b.save(using=database)
# Create bucket for all dates in the past
startdate = datetime.strptime(Parameter.objects.using(database).get(name="currentdate").value, "%Y-%m-%d %H:%M:%S")
curdate = startdate
BucketDetail(
bucket=b,
name='past',
startdate=datetime(2000, 1, 1),
enddate=curdate,
).save(using=database)
# Create daily buckets
limit = curdate + timedelta(min_day_horizon)
while curdate < limit or curdate.strftime("%w") != '0':
BucketDetail(
bucket=b,
name=str(curdate.date()),
startdate=curdate,
enddate=curdate + timedelta(1)
).save(using=database)
curdate = curdate + timedelta(1)
# Create weekly buckets
limit = startdate + timedelta(min_week_horizon)
stop = False
while not stop:
enddate = curdate + timedelta(7)
if curdate > limit and curdate.month != enddate.month:
stop = True
enddate = datetime(enddate.year, enddate.month, 1)
BucketDetail(
bucket=b,
name=curdate.strftime("%y W%W"),
startdate=curdate,
enddate=enddate
).save(using=database)
curdate = enddate
# Create monthly buckets
limit = startdate + timedelta(min_month_horizon)
while curdate < limit:
enddate = curdate + timedelta(32)
enddate = datetime(enddate.year, enddate.month, 1)
BucketDetail(
bucket=b,
name=curdate.strftime("%b %y"),
startdate=curdate,
enddate=enddate
).save(using=database)
curdate = enddate
transaction.commit(using=database)
finally:
transaction.rollback(using=database)
settings.DEBUG = tmp_debug
transaction.leave_transaction_management(using=database)
def handle(self, **options):
# Make sure the debug flag is not set!
# When it is set, the django database wrapper collects a list of all sql
# statements executed and their timings. This consumes plenty of memory
# and cpu time.
tmp_debug = settings.DEBUG
settings.DEBUG = False
# Pick up the options
if 'start' in options:
start = options['start'] or '2011-1-1'
else:
start = '2011-1-1'
if 'end' in options:
end = options['end'] or '2019-1-1'
else:
end = '2019-1-1'
if 'weekstart' in options:
weekstart = int(options['weekstart'])
if weekstart < 0 or weekstart > 6:
raise CommandError("Invalid weekstart %s" % weekstart)
else:
weekstart = 1
if 'database' in options:
database = options['database'] or DEFAULT_DB_ALIAS
else:
database = DEFAULT_DB_ALIAS
if database not in settings.DATABASES:
raise CommandError("No database settings known for '%s'" %
database)
if 'user' in options and options['user']:
try:
user = User.objects.all().using(database).get(
username=options['user'])
except:
raise CommandError("User '%s' not found" % options['user'])
else:
user = None
now = datetime.now()
task = None
try:
# Initialize the task
if 'task' in options and options['task']:
try:
task = Task.objects.all().using(database).get(
pk=options['task'])
except:
raise CommandError("Task identifier not found")
if task.started or task.finished or task.status != "Waiting" or task.name != 'generate buckets':
raise CommandError("Invalid task identifier")
task.status = '0%'
task.started = now
else:
task = Task(name='generate buckets',
submitted=now,
started=now,
status='0%',
user=user,
arguments="--start=%s --end=%s --weekstart=%s" %
(start, end, weekstart))
task.save(using=database)
# Validate the date arguments
try:
curdate = datetime.strptime(start, '%Y-%m-%d')
enddate = datetime.strptime(end, '%Y-%m-%d')
except Exception as e:
raise CommandError("Date is not matching format YYYY-MM-DD")
with transaction.atomic(using=database, savepoint=False):
# Delete previous contents
connections[database].cursor().execute(
"delete from common_bucketdetail where bucket_id in ('year','quarter','month','week','day')"
)
connections[database].cursor().execute(
"delete from common_bucket where name in ('year','quarter','month','week','day')"
)
# Create buckets
y = Bucket(name='year',
description='Yearly time buckets',
level=1)
q = Bucket(name='quarter',
description='Quarterly time buckets',
level=2)
m = Bucket(name='month',
description='Monthly time buckets',
level=3)
w = Bucket(name='week',
description='Weeky time buckets',
level=4)
d = Bucket(name='day',
description='Daily time buckets',
level=5)
y.save(using=database)
q.save(using=database)
m.save(using=database)
w.save(using=database)
d.save(using=database)
# Loop over all days in the chosen horizon
prev_year = None
prev_quarter = None
prev_month = None
prev_week = None
while curdate < enddate:
month = int(curdate.strftime(
"%m")) # an integer in the range 1 - 12
quarter = (month -
1) // 3 + 1 # an integer in the range 1 - 4
year = int(curdate.strftime("%Y"))
dayofweek = int(curdate.strftime(
"%w")) # day of the week, 0 = sunday, 1 = monday, ...
year_start = datetime(year, 1, 1)
year_end = datetime(year + 1, 1, 1)
week_start = curdate - timedelta((dayofweek + 6) % 7 + 1 -
weekstart)
week_end = curdate - timedelta((dayofweek + 6) % 7 - 6 -
weekstart)
if week_start < year_start:
week_start = year_start
if week_end > year_end:
week_end = year_end
# Create buckets
if year != prev_year:
prev_year = year
BucketDetail(bucket=y,
name=str(year),
startdate=year_start,
enddate=year_end).save(using=database)
if quarter != prev_quarter:
prev_quarter = quarter
BucketDetail(bucket=q,
name="%02d Q%s" % (year - 2000, quarter),
startdate=date(year, quarter * 3 - 2, 1),
enddate=date(
year + quarter // 4,
quarter * 3 + 1 - 12 * (quarter // 4),
1)).save(using=database)
if month != prev_month:
prev_month = month
BucketDetail(
bucket=m,
name=curdate.strftime("%b %y"),
startdate=date(year, month, 1),
enddate=date(year + month // 12,
month + 1 - 12 * (month // 12), 1),
).save(using=database)
if week_start != prev_week:
prev_week = week_start
BucketDetail(
bucket=w,
name=curdate.strftime("%y W%W"),
startdate=week_start,
enddate=week_end,
).save(using=database)
BucketDetail(
bucket=d,
name=str(curdate.date()),
startdate=curdate,
enddate=curdate + timedelta(1),
).save(using=database)
# Next date
curdate = curdate + timedelta(1)
# Log success
task.status = 'Done'
task.finished = datetime.now()
except Exception as e:
if task:
task.status = 'Failed'
task.message = '%s' % e
task.finished = datetime.now()
raise e
finally:
if task:
task.save(using=database)
settings.DEBUG = tmp_debug
def updateTelescope(min_day_horizon=10, min_week_horizon=40, min_month_horizon=730, database=DEFAULT_DB_ALIAS):
'''
Update for the telescopic horizon.
The first argument specifies the minimum number of daily buckets. Additional
daily buckets will be appended till we come to a monday. At that date weekly
buckets are starting.
The second argument specifies the minimum horizon with weeks before the
monthly buckets. The last weekly bucket can be a partial one: starting on
monday and ending on the first day of the next calendar month.
'''
# Make sure the debug flag is not set!
# When it is set, the django database wrapper collects a list of all sql
# statements executed and their timings. This consumes plenty of memory
# and cpu time.
tmp_debug = settings.DEBUG
settings.DEBUG = False
transaction.enter_transaction_management(using=database)
try:
# Delete previous contents
connections[database].cursor().execute(
"delete from common_bucketdetail where bucket_id = 'telescope'"
)
# Create bucket
try:
b = Bucket.objects.using(database).get(name='telescope')
except Bucket.DoesNotExist:
b = Bucket(name='telescope', description='Time buckets with decreasing granularity')
b.save(using=database)
# Create bucket for all dates in the past
startdate = datetime.strptime(Parameter.objects.using(database).get(name="currentdate").value, "%Y-%m-%d %H:%M:%S")
curdate = startdate
BucketDetail(
bucket=b,
name='past',
startdate=datetime(2000, 1, 1),
enddate=curdate,
).save(using=database)
# Create daily buckets
limit = curdate + timedelta(min_day_horizon)
while curdate < limit or curdate.strftime("%w") != '0':
BucketDetail(
bucket=b,
name=str(curdate.date()),
startdate=curdate,
enddate=curdate + timedelta(1)
).save(using=database)
curdate = curdate + timedelta(1)
# Create weekly buckets
limit = startdate + timedelta(min_week_horizon)
stop = False
while not stop:
enddate = curdate + timedelta(7)
if curdate > limit and curdate.month != enddate.month:
stop = True
enddate = datetime(enddate.year, enddate.month, 1)
BucketDetail(
bucket=b,
name=curdate.strftime("%y W%W"),
startdate=curdate,
enddate=enddate
).save(using=database)
curdate = enddate
# Create monthly buckets
limit = startdate + timedelta(min_month_horizon)
while curdate < limit:
enddate = curdate + timedelta(32)
enddate = datetime(enddate.year, enddate.month, 1)
BucketDetail(
bucket=b,
name=curdate.strftime("%b %y"),
startdate=curdate,
enddate=enddate
).save(using=database)
curdate = enddate
transaction.commit(using=database)
finally:
transaction.rollback(using=database)
settings.DEBUG = tmp_debug
transaction.leave_transaction_management(using=database)
