def get_easy_file_downloads_by_month(self, **extra_filters):
file_counts_by_month = self.get_easy_file_downloads_counts(
**extra_filters)
running_total = self.get_easy_file_downloads_running_total(
**extra_filters)
noncumulative = self.noncumulative
formatted_records = [] # move from a queryset to a []
for d in file_counts_by_month:
year_month = d['yyyy_mm'][:7]
year = int(d['yyyy_mm'][:4])
try:
month = int(d['yyyy_mm'][5:7])
except:
return StatsResult.build_error_result(
"in converting %s (month) into an integer (in get_easy_dataset_count_by_month)"
% d['yyyy_mm'][5:7])
fmt_rec = OrderedDict()
fmt_rec['yyyy_mm'] = year_month
fmt_rec['count'] = d['count']
# running total
running_total += d['count']
if noncumulative:
fmt_rec['running_total'] = d['count']
else:
fmt_rec['running_total'] = running_total
# Add year and month numbers
fmt_rec['year_num'] = year
fmt_rec['month_num'] = month
# Add month name
month_name_found, month_name_short = get_month_name_abbreviation(
month)
if month_name_found:
assume_month_name_found, fmt_rec[
'month_name'] = get_month_name(month)
fmt_rec['month_name_short'] = month_name_short
else:
logging.warning(
"no month name found for month %d (get_easy_file_downloads_by_month)"
% month)
formatted_records.append(fmt_rec)
data_dict = OrderedDict()
data_dict['total_downloads'] = running_total
data_dict['record_count'] = len(formatted_records)
data_dict['records'] = formatted_records
return StatsResult.build_success_result(data_dict, None)
def get_easy_counts_by_month(self, ds_counts_by_month, running_total,
noncumulative):
formatted_records = []
for d in ds_counts_by_month:
year_month = d['yyyy_mm'][:7]
year = int(d['yyyy_mm'][:4])
try:
month = int(d['yyyy_mm'][5:7])
except:
return StatsResult.build_error_result(
"in converting %s (month) into an integer (in get_easy_dataset_count_by_month)"
% d['yyyy_mm'][5:7])
fmt_dict = OrderedDict()
fmt_dict['yyyy_mm'] = year_month
fmt_dict['count'] = d['count']
# running total
running_total += d['count']
if noncumulative:
fmt_dict['running_total'] = d['count']
else:
fmt_dict['running_total'] = running_total
# Add year and month numbers
fmt_dict['year_num'] = year
fmt_dict['month_num'] = month
# Add month name
month_name_found, month_name_short = get_month_name_abbreviation(
month)
if month_name_found:
assume_month_name_found, fmt_dict[
'month_name'] = get_month_name(month)
fmt_dict['month_name_short'] = month_name_short
else:
logging.warning(
"no month name found for month %d (get_easy_dataset_count_by_month)"
% month)
# Add formatted record
formatted_records.append(fmt_dict)
data_dict = OrderedDict()
data_dict['record_count'] = len(formatted_records)
data_dict['records'] = formatted_records
return StatsResult.build_success_result(data_dict, None)
def get_dataset_count_by_month(self,
date_param=DVOBJECT_CREATEDATE_ATTR,
**extra_filters):
"""
Return dataset counts by month
"""
# Was an error found earlier?
#
if self.was_error_found():
return self.get_error_msg_return()
# -----------------------------------
# (1) Build query filters
# -----------------------------------
# Exclude records where dates are null
# - e.g. a record may not have a publication date
if date_param == DVOBJECT_CREATEDATE_ATTR:
exclude_params = {}
else:
exclude_params = {'%s__isnull' % date_param: True}
# Retrieve the date parameters
#
filter_params = self.get_date_filter_params()
# Add extra filters from kwargs
#
if extra_filters:
for k, v in extra_filters.items():
filter_params[k] = v
# -----------------------------------
# (2) Construct query
# -----------------------------------
# add exclude filters date filters
#
ds_counts_by_month = Dataset.objects.select_related('dvobject'\
).exclude(**exclude_params\
).filter(**filter_params)
# annotate query adding "month_year" and "cnt"
#
ds_counts_by_month = ds_counts_by_month.annotate(\
yyyy_mm=TruncYearMonth('%s' % date_param)\
).values('yyyy_mm'\
).annotate(count=models.Count('dvobject_id')\
).values('yyyy_mm', 'count'\
).order_by('%syyyy_mm' % self.time_sort)
# store query string
sql_query = str(ds_counts_by_month.query)
# -----------------------------------
# (3) Format results
# -----------------------------------
# hold the running total count
running_total = self.get_dataset_count_start_point(**extra_filters)
formatted_records = [] # move from a queryset to a []
for d in ds_counts_by_month:
fmt_dict = OrderedDict()
fmt_dict['yyyy_mm'] = d['yyyy_mm'].strftime('%Y-%m')
fmt_dict['count'] = d['count']
# running total
running_total += d['count']
fmt_dict['running_total'] = running_total
# d['month_year'] = d['yyyy_mm'].strftime('%Y-%m')
# Add year and month numbers
fmt_dict['year_num'] = d['yyyy_mm'].year
fmt_dict['month_num'] = d['yyyy_mm'].month
# Add month name
month_name_found, month_name_short = get_month_name_abbreviation(
d['yyyy_mm'].month)
if month_name_found:
assume_month_name_found, fmt_dict[
'month_name'] = get_month_name(d['yyyy_mm'].month)
fmt_dict['month_name_short'] = month_name_short
else:
# Log it!!!!!!
pass
# Add formatted record
formatted_records.append(fmt_dict)
data_dict = OrderedDict()
data_dict['record_count'] = len(formatted_records)
data_dict['records'] = formatted_records
return StatsResult.build_success_result(data_dict, sql_query)
def get_dataverse_counts_by_month(self, date_param=DVOBJECT_CREATEDATE_ATTR, **extra_filters):
"""
Return Dataverse counts by month
"""
# Was an error found earlier?
#
if self.was_error_found():
return self.get_error_msg_return()
# -----------------------------------
# (1) Build query filters
# -----------------------------------
# Exclude records where dates are null
# - e.g. a record may not have a publication date
exclude_params = { '%s__isnull' % date_param : True}
if self.include_harvested:
exclude_params['dvobject__id__in'] = self.get_harvested_dataverse_ids()
# Retrieve the date parameters
#
filter_params = self.get_date_filter_params()
# Add extra filters from kwargs
#
if extra_filters:
for k, v in extra_filters.items():
filter_params[k] = v
# -----------------------------------
# (2) Construct query
# -----------------------------------
# add exclude filters date filters
#
dv_counts_by_month = Dataverse.objects.select_related('dvobject'\
).exclude(**exclude_params\
).filter(**filter_params)
# annotate query adding "month_year" and "count"
#
dv_counts_by_month = dv_counts_by_month.annotate(\
yyyy_mm=TruncYearMonth('%s' % date_param)\
).values('yyyy_mm'\
).annotate(count=models.Count('dvobject_id')\
).values('yyyy_mm', 'count'\
).order_by('%syyyy_mm' % self.time_sort)
# -----------------------------------
# (2a) Get SQL query string
# -----------------------------------
sql_query = str(dv_counts_by_month.query)
# -----------------------------------
# (3) Format results
# -----------------------------------
# hold the running total count
running_total = self.get_dataverse_count_start_point(**extra_filters)
formatted_records = [] # move from a queryset to a []
for d in dv_counts_by_month:
rec_fmt = OrderedDict()
# change the datetime object to a string
rec_fmt['yyyy_mm'] = d['yyyy_mm'].strftime('%Y-%m')
rec_fmt['count'] = d['count']
# running total
running_total += d['count']
rec_fmt['running_total'] = running_total
# d['month_year'] = d['yyyy_mm'].strftime('%Y-%m')
# Add year and month numbers
rec_fmt['year_num'] = d['yyyy_mm'].year
rec_fmt['month_num'] = d['yyyy_mm'].month
# Add month name
month_name_found, month_name_short = get_month_name_abbreviation(d['yyyy_mm'].month)
if month_name_found:
assume_month_name_found, rec_fmt['month_name'] = get_month_name(d['yyyy_mm'].month)
rec_fmt['month_name_short'] = month_name_short
else:
# Log it!!!!!!
pass
# Add formatted record
formatted_records.append(rec_fmt)
data_dict = OrderedDict()
data_dict['record_count'] = len(formatted_records)
data_dict['total_count'] = running_total
data_dict['records'] = formatted_records
return StatsResult.build_success_result(data_dict, sql_query)
def get_file_downloads_by_month(self, **extra_filters):
"""
Using the GuestBookResponse object, find the number of file
downloads per month
"""
if self.was_error_found():
return self.get_error_msg_return()
filter_params = self.get_date_filter_params(
date_var_name='responsetime')
filter_params.update(self.get_download_type_filter())
# Narrow down to specific Dataverses
filter_params.update(self.get_dataverse_params_for_guestbook())
if self.was_error_found():
return self.get_error_msg_return()
# Add extra filters, if they exist
count_pre_dv4_downloads = False
if extra_filters:
for k, v in extra_filters.items():
if k == INCLUDE_PRE_DV4_DOWNLOADS: # skip this param
count_pre_dv4_downloads = True
del extra_filters[k]
else:
filter_params[k] = v
file_counts_by_month = GuestBookResponse.objects.exclude(\
responsetime__isnull=True\
).filter(**filter_params\
).annotate(yyyy_mm=TruncYearMonth('responsetime')\
).values('yyyy_mm'\
).annotate(count=models.Count('id')\
).values('yyyy_mm', 'count'\
).order_by('%syyyy_mm' % self.time_sort)
#print 'file_counts_by_month.query', file_counts_by_month.query
sql_query = str(file_counts_by_month.query)
formatted_records = [] # move from a queryset to a []
if count_pre_dv4_downloads:
file_running_total = self.get_file_download_start_point_include_undated(
**extra_filters)
else:
file_running_total = self.get_file_download_start_point(
**extra_filters)
for d in file_counts_by_month:
fmt_rec = OrderedDict()
fmt_rec['yyyy_mm'] = d['yyyy_mm'].strftime('%Y-%m')
fmt_rec['count'] = d['count']
file_running_total += d['count']
fmt_rec['running_total'] = file_running_total
# d['month_year'] = d['yyyy_mm'].strftime('%Y-%m')
# Add year and month numbers
fmt_rec['year_num'] = d['yyyy_mm'].year
fmt_rec['month_num'] = d['yyyy_mm'].month
# Add month name
month_name_found, month_name_short = get_month_name_abbreviation(
d['yyyy_mm'].month)
if month_name_found:
assume_month_name_found, fmt_rec[
'month_name'] = get_month_name(d['yyyy_mm'].month)
fmt_rec['month_name_short'] = month_name_short
else:
# Log it!!!!!!
pass
formatted_records.append(fmt_rec)
data_dict = OrderedDict()
data_dict['total_downloads'] = file_running_total
data_dict['record_count'] = len(formatted_records)
data_dict['records'] = formatted_records
return StatsResult.build_success_result(data_dict, sql_query)
def get_file_count_by_month(self, date_param=DVOBJECT_CREATEDATE_ATTR, **extra_filters):
"""
File counts by month
"""
# Was an error found earlier?
#
if self.was_error_found():
return self.get_error_msg_return()
# -----------------------------------
# (1) Build query filters
# -----------------------------------
# Exclude records where dates are null
# - e.g. a record may not have a publication date
if date_param == DVOBJECT_CREATEDATE_ATTR:
exclude_params = {}
else:
exclude_params = { '%s__isnull' % date_param : True}
# Retrieve the date parameters
#
filter_params = self.get_date_filter_params()
# Add extra filters from kwargs
#
if extra_filters:
for k, v in extra_filters.items():
filter_params[k] = v
# -----------------------------------
# (2) Construct query
# -----------------------------------
# add exclude filters date filters
#
file_counts_by_month = Datafile.objects.select_related('dvobject'\
).exclude(**exclude_params\
).filter(**filter_params)
# annotate query adding "month_year" and "cnt"
#
file_counts_by_month = file_counts_by_month.annotate(\
yyyy_mm=TruncYearMonth('%s' % date_param)\
).values('yyyy_mm'\
).annotate(count=models.Count('dvobject_id')\
).annotate(bytes=models.Sum('filesize')\
).values('yyyy_mm', 'count', 'bytes'\
).order_by('%syyyy_mm' % self.time_sort)
sql_query = str(file_counts_by_month.query)
# -----------------------------------
# (3) Format results
# -----------------------------------
running_total = self.get_file_count_start_point(**extra_filters) # hold the running total count
total_bytes = 0
formatted_records = [] # move from a queryset to a []
for d in file_counts_by_month:
fmt_rec = OrderedDict()
fmt_rec['yyyy_mm'] = d['yyyy_mm'].strftime('%Y-%m')
fmt_rec['count'] = d['count']
fmt_rec['bytes'] = d['bytes']
fmt_rec['bytes_str'] = comma_sep_number(d['bytes'])
total_bytes += d['bytes']
# running total
running_total += d['count']
fmt_rec['running_total'] = running_total
# d['month_year'] = d['yyyy_mm'].strftime('%Y-%m')
# Add year and month numbers
fmt_rec['year_num'] = d['yyyy_mm'].year
fmt_rec['month_num'] = d['yyyy_mm'].month
# Add month name
month_name_found, month_name_short = get_month_name_abbreviation(d['yyyy_mm'].month)
if month_name_found:
assume_month_name_found, fmt_rec['month_name'] = get_month_name(d['yyyy_mm'].month)
fmt_rec['month_name_short'] = month_name_short
else:
# Log it!!!!!!
pass
# Add formatted record
formatted_records.append(fmt_rec)
data_dict = OrderedDict()
data_dict['record_count'] = len(formatted_records)
data_dict['records'] = formatted_records
data_dict['total_bytes'] = total_bytes
data_dict['total_bytes_str'] = comma_sep_number(total_bytes)
return StatsResult.build_success_result(data_dict, sql_query)
def get_file_downloads_by_month(self, **extra_filters):
"""
Using the GuestBookResponse object, find the number of file
downloads per month
"""
if self.was_error_found():
return self.get_error_msg_return()
filter_params = self.get_date_filter_params(date_var_name='responsetime')
filter_params.update(self.get_download_type_filter())
# Narrow down to specific Dataverses
filter_params.update(self.get_dataverse_params_for_guestbook())
if self.was_error_found():
return self.get_error_msg_return()
# Add extra filters, if they exist
count_pre_dv4_downloads = False
if extra_filters:
for k, v in extra_filters.items():
if k == INCLUDE_PRE_DV4_DOWNLOADS: # skip this param
count_pre_dv4_downloads = True
del extra_filters[k]
else:
filter_params[k] = v
file_counts_by_month = GuestBookResponse.objects.exclude(\
responsetime__isnull=True\
).filter(**filter_params\
).annotate(yyyy_mm=TruncYearMonth('responsetime')\
).values('yyyy_mm'\
).annotate(count=models.Count('id')\
).values('yyyy_mm', 'count'\
).order_by('%syyyy_mm' % self.time_sort)
#print 'file_counts_by_month.query', file_counts_by_month.query
sql_query = str(file_counts_by_month.query)
formatted_records = [] # move from a queryset to a []
if count_pre_dv4_downloads:
file_running_total = self.get_file_download_start_point_include_undated(**extra_filters)
else:
file_running_total = self.get_file_download_start_point(**extra_filters)
for d in file_counts_by_month:
fmt_rec = OrderedDict()
fmt_rec['yyyy_mm'] = d['yyyy_mm'].strftime('%Y-%m')
fmt_rec['count'] = d['count']
file_running_total += d['count']
fmt_rec['running_total'] = file_running_total
# d['month_year'] = d['yyyy_mm'].strftime('%Y-%m')
# Add year and month numbers
fmt_rec['year_num'] = d['yyyy_mm'].year
fmt_rec['month_num'] = d['yyyy_mm'].month
# Add month name
month_name_found, month_name_short = get_month_name_abbreviation( d['yyyy_mm'].month)
if month_name_found:
assume_month_name_found, fmt_rec['month_name'] = get_month_name(d['yyyy_mm'].month)
fmt_rec['month_name_short'] = month_name_short
else:
# Log it!!!!!!
pass
formatted_records.append(fmt_rec)
data_dict = OrderedDict()
data_dict['total_downloads'] = file_running_total
data_dict['record_count'] = len(formatted_records)
data_dict['records'] = formatted_records
return StatsResult.build_success_result(data_dict, sql_query)