def get_dataverse_count(self, **extra_filters): """ Return the Dataverse count -- a single number """ if self.was_error_found(): return self.get_error_msg_return() filter_params = self.get_date_filter_params() if extra_filters: for k, v in extra_filters.items(): filter_params[k] = v if self.include_harvested: q = Dataverse.objects.filter(**filter_params) else: q = Dataverse.objects.filter(**filter_params\ ).exclude(self.get_harvested_dataverse_ids()\ ) sql_query = str(q.query) data_dict = OrderedDict() data_dict['count'] = q.count() data_dict['count_string'] = "{:,}".format(data_dict['count']) return StatsResult.build_success_result(data_dict, sql_query)
def get_dataverse_count(self, **extra_filters): """ Return the Dataverse count -- a single number """ if self.was_error_found(): return self.get_error_msg_return() filter_params = self.get_date_filter_params() if extra_filters: for k, v in extra_filters.items(): filter_params[k] = v if self.include_harvested: q = Dataverse.objects.filter(**filter_params) else: q = Dataverse.objects.filter(**filter_params\ ).exclude(self.get_harvested_dataverse_ids()\ ) sql_query = str(q.query) data_dict = OrderedDict() data_dict['count'] = q.count() data_dict['count_string'] = "{:,}".format(data_dict['count']) return StatsResult.build_success_result(data_dict, sql_query)
def get_easy_file_downloads_by_month(self, **extra_filters): file_counts_by_month = self.get_easy_file_downloads_counts( **extra_filters) running_total = self.get_easy_file_downloads_running_total( **extra_filters) noncumulative = self.noncumulative formatted_records = [] # move from a queryset to a [] for d in file_counts_by_month: year_month = d['yyyy_mm'][:7] year = int(d['yyyy_mm'][:4]) try: month = int(d['yyyy_mm'][5:7]) except: return StatsResult.build_error_result( "in converting %s (month) into an integer (in get_easy_dataset_count_by_month)" % d['yyyy_mm'][5:7]) fmt_rec = OrderedDict() fmt_rec['yyyy_mm'] = year_month fmt_rec['count'] = d['count'] # running total running_total += d['count'] if noncumulative: fmt_rec['running_total'] = d['count'] else: fmt_rec['running_total'] = running_total # Add year and month numbers fmt_rec['year_num'] = year fmt_rec['month_num'] = month # Add month name month_name_found, month_name_short = get_month_name_abbreviation( month) if month_name_found: assume_month_name_found, fmt_rec[ 'month_name'] = get_month_name(month) fmt_rec['month_name_short'] = month_name_short else: logging.warning( "no month name found for month %d (get_easy_file_downloads_by_month)" % month) formatted_records.append(fmt_rec) data_dict = OrderedDict() data_dict['total_downloads'] = running_total data_dict['record_count'] = len(formatted_records) data_dict['records'] = formatted_records return StatsResult.build_success_result(data_dict, None)
def get_easy_counts_by_month(self, ds_counts_by_month, running_total, noncumulative): formatted_records = [] for d in ds_counts_by_month: year_month = d['yyyy_mm'][:7] year = int(d['yyyy_mm'][:4]) try: month = int(d['yyyy_mm'][5:7]) except: return StatsResult.build_error_result( "in converting %s (month) into an integer (in get_easy_dataset_count_by_month)" % d['yyyy_mm'][5:7]) fmt_dict = OrderedDict() fmt_dict['yyyy_mm'] = year_month fmt_dict['count'] = d['count'] # running total running_total += d['count'] if noncumulative: fmt_dict['running_total'] = d['count'] else: fmt_dict['running_total'] = running_total # Add year and month numbers fmt_dict['year_num'] = year fmt_dict['month_num'] = month # Add month name month_name_found, month_name_short = get_month_name_abbreviation( month) if month_name_found: assume_month_name_found, fmt_dict[ 'month_name'] = get_month_name(month) fmt_dict['month_name_short'] = month_name_short else: logging.warning( "no month name found for month %d (get_easy_dataset_count_by_month)" % month) # Add formatted record formatted_records.append(fmt_dict) data_dict = OrderedDict() data_dict['record_count'] = len(formatted_records) data_dict['records'] = formatted_records return StatsResult.build_success_result(data_dict, None)
def view_file_extensions_within_type(self, file_type=None): """View extensions for files based on their "Filemetadata.contenttype" value""" #file_type = 'data/various-formats' if file_type is None: # Retrieve list of **all** file names -- this could be too much! l = FileMetadata.objects.distinct('datafile__id', 'label'\ ).values_list('datafile__id', 'label') else: # Retrieve ids of Datafile filtered by "contenttype" ids = Datafile.objects.filter(contenttype=file_type).values_list( 'dvobject__id', flat=True) # Retrieve the names of these Datafiles via the FileMetadata object l = FileMetadata.objects.filter(datafile__in=ids\ ).distinct('datafile__id', 'label'\ ).values_list('datafile__id', 'label') # Convert the file names to file extensions ext_list = [splitext(info[1])[-1] for info in l] # Make a dict counting the extensions extension_counts = { } # {file extension : count, file ext : count, etc} for ext in ext_list: extension_counts[ext] = extension_counts.get(ext, 0) + 1 # Sort the counts in descending order--highest count first ext_pairs = extension_counts.items() ext_pairs = sorted(ext_pairs, key=lambda k: k[1], reverse=True) ext_list = [] total_count = sum(x[1] for x in ext_pairs) + 0.000 for ext_pair in ext_pairs: d = OrderedDict(extension=ext_pair[0]) d['count'] = ext_pair[1] d['total_count'] = int(total_count) d['percent_string'] = '{0:.3%}'.format(ext_pair[1] / total_count) ext_list.append(d) data_dict = OrderedDict(number_unique_extensions=len(ext_pairs)) data_dict['total_file_count'] = int(total_count) data_dict['record_count'] = len(ext_list) data_dict['records'] = ext_list data_dict['all_dv_files'] = Datafile.objects.all().count() data_dict['percent_unknown'] = '{0:.3%}'.format( total_count / data_dict['all_dv_files']) return StatsResult.build_success_result(data_dict)
def get_stats_result(self, request): """Return the StatsResult object for this statistic""" dv_id = self.kwargs.get('ds_id', None) if dv_id is None: return StatsResult.build_error_result("No Dataset id specified", 400) # Get the latest version dataset_version = get_latest_dataset_version(dv_id) if dataset_version is None: return StatsResult.build_error_result('No published Dataset with id: %s' % dv_id, 404) dataset_as_json = DatasetSerializer(dataset_version).as_json() return StatsResult.build_success_result(dataset_as_json)
def view_file_extensions_within_type(self, file_type=None): """View extensions for files based on their "Filemetadata.contenttype" value""" #file_type = 'data/various-formats' if file_type is None: # Retrieve list of **all** file names -- this could be too much! l = FileMetadata.objects.distinct('datafile__id', 'label'\ ).values_list('datafile__id', 'label') else: # Retrieve ids of Datafile filtered by "contenttype" ids = Datafile.objects.filter(contenttype=file_type).values_list('dvobject__id', flat=True) # Retrieve the names of these Datafiles via the FileMetadata object l = FileMetadata.objects.filter(datafile__in=ids\ ).distinct('datafile__id', 'label'\ ).values_list('datafile__id', 'label') # Convert the file names to file extensions ext_list = [splitext(info[1])[-1] for info in l] # Make a dict counting the extensions extension_counts = {} # {file extension : count, file ext : count, etc} for ext in ext_list: extension_counts[ext] = extension_counts.get(ext, 0) + 1 # Sort the counts in descending order--highest count first ext_pairs = extension_counts.items() ext_pairs = sorted(ext_pairs, key=lambda k: k[1], reverse=True) ext_list = [] total_count = sum(x[1] for x in ext_pairs) + 0.000 for ext_pair in ext_pairs: d = OrderedDict(extension=ext_pair[0]) d['count'] = ext_pair[1] d['total_count'] = int(total_count) d['percent_string'] = '{0:.3%}'.format(ext_pair[1] / total_count) ext_list.append(d) data_dict = OrderedDict(number_unique_extensions=len(ext_pairs)) data_dict['total_file_count'] = int(total_count) data_dict['record_count'] = len(ext_list) data_dict['records'] = ext_list data_dict['all_dv_files'] = Datafile.objects.all().count() data_dict['percent_unknown'] = '{0:.3%}'.format(total_count/data_dict['all_dv_files']) return StatsResult.build_success_result(data_dict)
def get_stats_result(self, request): """Return the StatsResult object for this statistic""" #dv_id = request.GET.get('id', None) alias = self.kwargs.get('alias', None) if alias is None: return StatsResult.build_error_result("No Dataverse 'alias' specified", 400) try: selected_dv = Dataverse.objects.select_related('dvobject').get(\ alias=alias,\ dvobject__publicationdate__isnull=False) except Dataverse.DoesNotExist: return StatsResult.build_error_result('No published Dataverse with alias: %s' % alias, 404) dataverse_as_json = DataverseSerializer(selected_dv).as_json() return StatsResult.build_success_result(dataverse_as_json)
def get_stats_result(self, request): """Return the StatsResult object for this statistic""" dv_id = self.kwargs.get('ds_id', None) if dv_id is None: return StatsResult.build_error_result("No Dataset id specified", 400) # Get the latest version dataset_version = get_latest_dataset_version(dv_id) if dataset_version is None: return StatsResult.build_error_result( 'No published Dataset with id: %s' % dv_id, 404) dataset_as_json = DatasetSerializer(dataset_version).as_json() return StatsResult.build_success_result(dataset_as_json)
def get_dataset_category_counts(self, **extra_filters): """Dataset counts by subjet""" # Was an error found earlier? # if self.was_error_found(): return self.get_error_msg_return() if EASY_STATISTICS: ds_values = self.get_easy_dataset_category_counts() else: ds_values = self.get_dataverse_dataset_subject_counts( **extra_filters) # ----------------------------- # Iterate through the vocab values, # process the totals, calculate percentage # ----------------------------- running_total = 0 formatted_records = [] # move from a queryset to a [] total_count = sum([rec['cnt'] for rec in ds_values]) + 0.00 for info in ds_values: rec = OrderedDict() rec['category'] = info['category'] # count rec['count'] = info['cnt'] rec['total_count'] = int(total_count) # percent float_percent = info['cnt'] / total_count rec['percent_string'] = '{0:.1%}'.format(float_percent) rec['percent_number'] = float("%.3f" % (float_percent)) # total count formatted_records.append(rec) data_dict = OrderedDict() data_dict['record_count'] = len(formatted_records) data_dict['records'] = formatted_records return StatsResult.build_success_result(data_dict)
def get_total_file_downloads(self, **extra_filters): """ Get the total file download count """ if self.was_error_found(): return self.get_error_msg_return() filter_params = self.get_date_filter_params(date_var_name='responsetime') filter_params.update(self.get_download_type_filter()) # Narrow down to specific Dataverses filter_params.update(self.get_dataverse_params_for_guestbook()) if self.was_error_found(): return self.get_error_msg_return() # Add extra filters, if they exist count_pre_dv4_downloads = False if extra_filters: for k, v in extra_filters.items(): if k == INCLUDE_PRE_DV4_DOWNLOADS: # skip this param count_pre_dv4_downloads = True del extra_filters[k] else: filter_params[k] = v if count_pre_dv4_downloads: exclude_params = {} else: exclude_params = dict(responsetime__isnull=True) q = GuestBookResponse.objects.exclude(**exclude_params\ ).filter(**filter_params) sql_query = str(q.query) data_dict = OrderedDict() data_dict['count'] = q.count() data_dict['count_string'] = "{:,}".format(data_dict['count']) return StatsResult.build_success_result(data_dict, sql_query)
def get_total_file_downloads(self, **extra_filters): """ Get the total file download count """ if self.was_error_found(): return self.get_error_msg_return() filter_params = self.get_date_filter_params( date_var_name='responsetime') filter_params.update(self.get_download_type_filter()) # Narrow down to specific Dataverses filter_params.update(self.get_dataverse_params_for_guestbook()) if self.was_error_found(): return self.get_error_msg_return() # Add extra filters, if they exist count_pre_dv4_downloads = False if extra_filters: for k, v in extra_filters.items(): if k == INCLUDE_PRE_DV4_DOWNLOADS: # skip this param count_pre_dv4_downloads = True del extra_filters[k] else: filter_params[k] = v if count_pre_dv4_downloads: exclude_params = {} else: exclude_params = dict(responsetime__isnull=True) q = GuestBookResponse.objects.exclude(**exclude_params\ ).filter(**filter_params) sql_query = str(q.query) data_dict = OrderedDict() data_dict['count'] = q.count() data_dict['count_string'] = "{:,}".format(data_dict['count']) return StatsResult.build_success_result(data_dict, sql_query)
def get_dataset_count(self, **extra_filters): """ Return the Dataset count """ if self.was_error_found(): return self.get_error_msg_return() filter_params = self.get_date_filter_params() if extra_filters: for k, v in extra_filters.items(): filter_params[k] = v q = Dataset.objects.filter(**filter_params) sql_query = str(q.query) data_dict = OrderedDict() data_dict['count'] = q.count() data_dict['count_string'] = "{:,}".format(data_dict['count']) return StatsResult.build_success_result(data_dict, sql_query)
def get_datafile_count(self, **extra_filters): """ Return the Datafile count """ if self.was_error_found(): return self.get_error_msg_return() filter_params = self.get_date_filter_params() # Add extra filters, if they exist if extra_filters: for k, v in extra_filters.items(): filter_params[k] = v q = Datafile.objects.filter(**filter_params) sql_query = str(q.query) data_dict = OrderedDict() data_dict['count'] = q.count() data_dict['count_string'] = "{:,}".format(data_dict['count']) return StatsResult.build_success_result(data_dict, sql_query)
def get_stats_result(self, request): """Return the StatsResult object for this statistic""" persistent_id = request.GET.get('persistentId', None) if persistent_id is None: return StatsResult.build_error_result( "No Dataset persistent id specified", 400) ds = Dataset.get_dataset_by_persistent_id(persistent_id) err_404 = 'No published dataset found for persistentId: %s' % persistent_id if ds is None or not ds.dvobject.publicationdate: return StatsResult.build_error_result(err_404, 404) # Get the latest version dataset_version = get_latest_dataset_version(ds.dvobject.id) if dataset_version is None: return StatsResult.build_error_result(err_404, 404) dataset_as_json = DatasetSerializer(dataset_version).as_json() return StatsResult.build_success_result(dataset_as_json)
def get_stats_result(self, request): """Return the StatsResult object for this statistic""" persistent_id = request.GET.get('persistentId', None) if persistent_id is None: return StatsResult.build_error_result("No Dataset persistent id specified", 400) ds = Dataset.get_dataset_by_persistent_id(persistent_id) err_404 = 'No published dataset found for persistentId: %s' % persistent_id if ds is None or not ds.dvobject.publicationdate: return StatsResult.build_error_result(err_404, 404) # Get the latest version dataset_version = get_latest_dataset_version(ds.dvobject.id) if dataset_version is None: return StatsResult.build_error_result(err_404, 404) dataset_as_json = DatasetSerializer(dataset_version).as_json() return StatsResult.build_success_result(dataset_as_json)
def get_dataverse_affiliation_counts(self, **extra_filters): """ Return Dataverse counts by affiliation Returns: dv_counts_by_affiliation": [ { "affiliation": "University of Oxford", "affiliation_count": 2, "total_count": 191, "percent_string": "1.0%" }, { "affiliation": "University of Illinois", "affiliation_count": 1, "total_count": 191, "percent_string": "0.5%" } ... ] """ if self.was_error_found(): return self.get_error_msg_return() # Retrieve the date parameters # filter_params = self.get_date_filter_params(DVOBJECT_CREATEDATE_ATTR) if extra_filters: for k, v in extra_filters.items(): filter_params[k] = v dataverse_counts_by_affil = Dataverse.objects.select_related('dvobject'\ ).filter(**filter_params\ ).values('affiliation'\ ).order_by('affiliation'\ ).annotate(affiliation_count=models.Count('affiliation')\ ).order_by('-affiliation_count') # ----------------------------------- # Get SQL query string # ----------------------------------- sql_query = str(dataverse_counts_by_affil.query) # Count all dataverses # total_count = sum([rec.get('affiliation_count', 0) for rec in dataverse_counts_by_affil]) total_count = total_count + 0.0 print 'dataverse_counts_by_affil', dataverse_counts_by_affil # Format the records, adding 'total_count' and 'percent_string' to each one # formatted_records = [] for rec in dataverse_counts_by_affil: if rec.get('affiliation_count', 0) > 0: fmt_dict = OrderedDict() affil_str = rec.get('affiliation', None) if affil_str is not None: affil_str = affil_str.encode('utf-8') fmt_dict['affiliation'] = affil_str fmt_dict['affiliation_count'] = rec.get('affiliation_count', 0) if total_count > 0: float_percent = rec.get('affiliation_count', 0) / total_count fmt_dict['total_count'] = int(total_count) fmt_dict['percent_string'] = '{0:.1%}'.format(float_percent) else: fmt_dict['total_count'] = 0 fmt_dict['percent_string'] = '0%' formatted_records.append(fmt_dict) data_dict = OrderedDict() data_dict['record_count'] = len(formatted_records) data_dict['records'] = formatted_records return StatsResult.build_success_result(data_dict, sql_query)
def get_dataverse_counts_by_type(self, exclude_uncategorized=True, **extra_filters): """ Return dataverse counts by 'dataversetype' Optional if a dataverse is uncategorized: - Specifying 'uncategorized_replacement_name' will set "UNCATEGORIZED" to another string Returns: { "dv_counts_by_type": [ { "dataversetype": "RESEARCH_PROJECTS", "type_count": 85, "total_count": 356, "percent_string": "23.9%" }, { "dataversetype": "TEACHING_COURSES", "type_count": 10, "total_count": 356, "percent_string": "2.8%" } ... etc ] } """ if self.was_error_found(): return self.get_error_msg_return() # Retrieve the date parameters # filter_params = self.get_date_filter_params(DVOBJECT_CREATEDATE_ATTR) # Add extra filters if extra_filters: for k, v in extra_filters.items(): filter_params[k] = v if exclude_uncategorized: exclude_params = dict(dataversetype=DATAVERSE_TYPE_UNCATEGORIZED) else: exclude_params = {} dataverse_counts_by_type = Dataverse.objects.select_related('dvobject'\ ).filter(**filter_params\ ).exclude(**exclude_params\ ).values('dataversetype'\ ).order_by('dataversetype'\ ).annotate(type_count=models.Count('dataversetype')\ ).order_by('-type_count') # ----------------------------------- # Get SQL query string # ----------------------------------- sql_query = str(dataverse_counts_by_type.query) # Count all dataverses # total_count = sum([rec.get('type_count', 0) for rec in dataverse_counts_by_type]) total_count = total_count + 0.0 # Format the records, adding 'total_count' and 'percent_string' to each one # formatted_records = [] for rec in dataverse_counts_by_type: fmt_dict = OrderedDict() fmt_dict['dataversetype'] = rec['dataversetype'] fmt_dict['dataversetype_label'] = rec['dataversetype'].replace('_', ' ') fmt_dict['type_count'] = rec.get('type_count', 0) if total_count > 0: float_percent = rec.get('type_count', 0) / total_count fmt_dict['total_count'] = int(total_count) fmt_dict['percent_string'] = '{0:.1%}'.format(float_percent) else: fmt_dict['total_count'] = 0 fmt_dict['percent_string'] = '0%' formatted_records.append(fmt_dict) data_dict = OrderedDict() data_dict['record_count'] = len(formatted_records) data_dict['records'] = formatted_records return StatsResult.build_success_result(data_dict, sql_query)
def get_dataverse_counts_by_month(self, date_param=DVOBJECT_CREATEDATE_ATTR, **extra_filters): """ Return Dataverse counts by month """ # Was an error found earlier? # if self.was_error_found(): return self.get_error_msg_return() # ----------------------------------- # (1) Build query filters # ----------------------------------- # Exclude records where dates are null # - e.g. a record may not have a publication date exclude_params = { '%s__isnull' % date_param : True} if self.include_harvested: exclude_params['dvobject__id__in'] = self.get_harvested_dataverse_ids() # Retrieve the date parameters # filter_params = self.get_date_filter_params() # Add extra filters from kwargs # if extra_filters: for k, v in extra_filters.items(): filter_params[k] = v # ----------------------------------- # (2) Construct query # ----------------------------------- # add exclude filters date filters # dv_counts_by_month = Dataverse.objects.select_related('dvobject'\ ).exclude(**exclude_params\ ).filter(**filter_params) # annotate query adding "month_year" and "count" # dv_counts_by_month = dv_counts_by_month.annotate(\ yyyy_mm=TruncYearMonth('%s' % date_param)\ ).values('yyyy_mm'\ ).annotate(count=models.Count('dvobject_id')\ ).values('yyyy_mm', 'count'\ ).order_by('%syyyy_mm' % self.time_sort) # ----------------------------------- # (2a) Get SQL query string # ----------------------------------- sql_query = str(dv_counts_by_month.query) # ----------------------------------- # (3) Format results # ----------------------------------- # hold the running total count running_total = self.get_dataverse_count_start_point(**extra_filters) formatted_records = [] # move from a queryset to a [] for d in dv_counts_by_month: rec_fmt = OrderedDict() # change the datetime object to a string rec_fmt['yyyy_mm'] = d['yyyy_mm'].strftime('%Y-%m') rec_fmt['count'] = d['count'] # running total running_total += d['count'] rec_fmt['running_total'] = running_total # d['month_year'] = d['yyyy_mm'].strftime('%Y-%m') # Add year and month numbers rec_fmt['year_num'] = d['yyyy_mm'].year rec_fmt['month_num'] = d['yyyy_mm'].month # Add month name month_name_found, month_name_short = get_month_name_abbreviation(d['yyyy_mm'].month) if month_name_found: assume_month_name_found, rec_fmt['month_name'] = get_month_name(d['yyyy_mm'].month) rec_fmt['month_name_short'] = month_name_short else: # Log it!!!!!! pass # Add formatted record formatted_records.append(rec_fmt) data_dict = OrderedDict() data_dict['record_count'] = len(formatted_records) data_dict['total_count'] = running_total data_dict['records'] = formatted_records return StatsResult.build_success_result(data_dict, sql_query)
def get_file_counts_per_dataset_latest_versions(self, **extra_filters): """ Get binning stats for the number of files in each Dataset. For the counts, only use the LATEST DatasetVersion """ # Get the correct DatasetVersion ids as a filter parameter # latest_dsv_ids = self.get_dataset_version_ids(**extra_filters) filter_params = dict(datasetversion__id__in=latest_dsv_ids) # Make query # ds_version_counts = FileMetadata.objects.filter(**filter_params\ ).annotate(dsv_id=F('datasetversion__id'),\ ).values('dsv_id',\ ).annotate(cnt=models.Count('datafile__id')\ ).values('dsv_id', 'cnt'\ ).order_by('-cnt') # Convert to Dataframe # df = pd.DataFrame(list(ds_version_counts), columns = ['dsv_id', 'cnt']) # Get the list of bins # high_num = high_num=df['cnt'].max() + self.bin_size bins = self.get_bin_list(step=self.bin_size, low_num=0, high_num=high_num+self.bin_size) # Add a new column, assigning each file count to a bin # df['bin_label'] = pd.cut(df['cnt'], bins) # Count the occurrence of each bin # bin_count_series = pd.value_counts(df['bin_label']) # Make the Series into a new DataFrame # df_bins = pd.DataFrame(dict(bin=bin_count_series.index,\ count=bin_count_series.values)) # Add a sort key # (0, 20] -> 0 # (20, 30] -> 20 # etc df_bins['sort_key'] = df_bins['bin'].apply(lambda x: int(x[1:-1].split(',')[0])) df_bins['bin_start_inclusive'] = df_bins['sort_key'] df_bins['bin_end'] = df_bins['bin'].apply(lambda x: int(x[1:-1].split(',')[1])) # Add a formatted string # (0, 20] -> 0 to 20 # (20, 30] -> 20 to 30 # etc df_bins['bin_str'] = df_bins['bin'].apply(lambda x: x[1:-1].replace(', ', ' to ')) # Sort the bins # df_bins = df_bins.sort('sort_key') #msgt(df_bins) # If appropriate, skip empty bins, e.g. remove 0 counts # if self.skip_empty_bins: df_bins = df_bins.query('count != 0') #msg(df_bins) # Return as python dict # # bit expensive but want orderedDict formatted_records_json = df_bins.to_json(orient='records') formatted_records = json.loads(formatted_records_json, object_pairs_hook=OrderedDict) data_dict = OrderedDict() data_dict['record_count'] = len(formatted_records) data_dict['records'] = formatted_records return StatsResult.build_success_result(data_dict)
def get_datafile_content_type_counts(self, **extra_filters): """ Return datafile counts by 'content type' "datafile_content_type_counts": [ { "total_count": 1584, "contenttype": "text/tab-separated-values", "type_count": 187, "percent_string": "11.8%" }, { "total_count": 1584, "contenttype": "image/jpeg", "type_count": 182, "percent_string": "11.5%" }, { "total_count": 1584, "contenttype": "text/plain", "type_count": 147, "percent_string": "9.3%" } ] """ if self.was_error_found(): return self.get_error_msg_return() # Retrieve the date parameters # filter_params = self.get_date_filter_params(DVOBJECT_CREATEDATE_ATTR) # Add extra filters if extra_filters: for k, v in extra_filters.items(): filter_params[k] = v datafile_counts_by_type = Datafile.objects.select_related('dvobject'\ ).filter(**filter_params\ ).values('contenttype'\ ).order_by('contenttype'\ ).annotate(type_count=models.Count('contenttype')\ ).order_by('-type_count') sql_query = str(datafile_counts_by_type.query) # Count all dataverses # total_count = sum([rec.get('type_count', 0) for rec in datafile_counts_by_type]) total_count = total_count + 0.0 # Format the records, adding 'total_count' and 'percent_string' to each one # formatted_records = [] #num = 0 for rec in datafile_counts_by_type: if total_count > 0: fmt_dict = OrderedDict() fmt_dict['contenttype'] = rec['contenttype'] # short contenttype contenttype_parts = rec['contenttype'].split('/') if len(contenttype_parts) > 1: fmt_dict['short_content_type'] = '/'.join(contenttype_parts[1:]) else: fmt_dict['short_content_type'] = rec['contenttype'] fmt_dict['type_count'] = rec.get('type_count', 0) float_percent = fmt_dict['type_count'] / total_count fmt_dict['total_count'] = int(total_count) fmt_dict['percent_string'] = '{0:.1%}'.format(float_percent) formatted_records.append(fmt_dict) data_dict = OrderedDict() data_dict['record_count'] = len(formatted_records) data_dict['records'] = formatted_records return StatsResult.build_success_result(data_dict, sql_query)
def get_datafile_content_type_counts(self, **extra_filters): """ Return datafile counts by 'content type' "datafile_content_type_counts": [ { "total_count": 1584, "contenttype": "text/tab-separated-values", "type_count": 187, "percent_string": "11.8%" }, { "total_count": 1584, "contenttype": "image/jpeg", "type_count": 182, "percent_string": "11.5%" }, { "total_count": 1584, "contenttype": "text/plain", "type_count": 147, "percent_string": "9.3%" } ] """ if self.was_error_found(): return self.get_error_msg_return() # Retrieve the date parameters # filter_params = self.get_date_filter_params(DVOBJECT_CREATEDATE_ATTR) # Add extra filters if extra_filters: for k, v in extra_filters.items(): filter_params[k] = v datafile_counts_by_type = Datafile.objects.select_related('dvobject'\ ).filter(**filter_params\ ).values('contenttype'\ ).order_by('contenttype'\ ).annotate(type_count=models.Count('contenttype')\ ).order_by('-type_count') sql_query = str(datafile_counts_by_type.query) # Count all dataverses # total_count = sum( [rec.get('type_count', 0) for rec in datafile_counts_by_type]) total_count = total_count + 0.0 # Format the records, adding 'total_count' and 'percent_string' to each one # formatted_records = [] #num = 0 for rec in datafile_counts_by_type: if total_count > 0: fmt_dict = OrderedDict() fmt_dict['contenttype'] = rec['contenttype'] # short contenttype contenttype_parts = rec['contenttype'].split('/') if len(contenttype_parts) > 1: fmt_dict['short_content_type'] = '/'.join( contenttype_parts[1:]) else: fmt_dict['short_content_type'] = rec['contenttype'] fmt_dict['type_count'] = rec.get('type_count', 0) float_percent = fmt_dict['type_count'] / total_count fmt_dict['total_count'] = int(total_count) fmt_dict['percent_string'] = '{0:.1%}'.format(float_percent) formatted_records.append(fmt_dict) data_dict = OrderedDict() data_dict['record_count'] = len(formatted_records) data_dict['records'] = formatted_records return StatsResult.build_success_result(data_dict, sql_query)
def get_file_downloads_by_month(self, **extra_filters): """ Using the GuestBookResponse object, find the number of file downloads per month """ if self.was_error_found(): return self.get_error_msg_return() filter_params = self.get_date_filter_params( date_var_name='responsetime') filter_params.update(self.get_download_type_filter()) # Narrow down to specific Dataverses filter_params.update(self.get_dataverse_params_for_guestbook()) if self.was_error_found(): return self.get_error_msg_return() # Add extra filters, if they exist count_pre_dv4_downloads = False if extra_filters: for k, v in extra_filters.items(): if k == INCLUDE_PRE_DV4_DOWNLOADS: # skip this param count_pre_dv4_downloads = True del extra_filters[k] else: filter_params[k] = v file_counts_by_month = GuestBookResponse.objects.exclude(\ responsetime__isnull=True\ ).filter(**filter_params\ ).annotate(yyyy_mm=TruncYearMonth('responsetime')\ ).values('yyyy_mm'\ ).annotate(count=models.Count('id')\ ).values('yyyy_mm', 'count'\ ).order_by('%syyyy_mm' % self.time_sort) #print 'file_counts_by_month.query', file_counts_by_month.query sql_query = str(file_counts_by_month.query) formatted_records = [] # move from a queryset to a [] if count_pre_dv4_downloads: file_running_total = self.get_file_download_start_point_include_undated( **extra_filters) else: file_running_total = self.get_file_download_start_point( **extra_filters) for d in file_counts_by_month: fmt_rec = OrderedDict() fmt_rec['yyyy_mm'] = d['yyyy_mm'].strftime('%Y-%m') fmt_rec['count'] = d['count'] file_running_total += d['count'] fmt_rec['running_total'] = file_running_total # d['month_year'] = d['yyyy_mm'].strftime('%Y-%m') # Add year and month numbers fmt_rec['year_num'] = d['yyyy_mm'].year fmt_rec['month_num'] = d['yyyy_mm'].month # Add month name month_name_found, month_name_short = get_month_name_abbreviation( d['yyyy_mm'].month) if month_name_found: assume_month_name_found, fmt_rec[ 'month_name'] = get_month_name(d['yyyy_mm'].month) fmt_rec['month_name_short'] = month_name_short else: # Log it!!!!!! pass formatted_records.append(fmt_rec) data_dict = OrderedDict() data_dict['total_downloads'] = file_running_total data_dict['record_count'] = len(formatted_records) data_dict['records'] = formatted_records return StatsResult.build_success_result(data_dict, sql_query)
def get_file_downloads_by_month(self, **extra_filters): """ Using the GuestBookResponse object, find the number of file downloads per month """ if self.was_error_found(): return self.get_error_msg_return() filter_params = self.get_date_filter_params(date_var_name='responsetime') filter_params.update(self.get_download_type_filter()) # Narrow down to specific Dataverses filter_params.update(self.get_dataverse_params_for_guestbook()) if self.was_error_found(): return self.get_error_msg_return() # Add extra filters, if they exist count_pre_dv4_downloads = False if extra_filters: for k, v in extra_filters.items(): if k == INCLUDE_PRE_DV4_DOWNLOADS: # skip this param count_pre_dv4_downloads = True del extra_filters[k] else: filter_params[k] = v file_counts_by_month = GuestBookResponse.objects.exclude(\ responsetime__isnull=True\ ).filter(**filter_params\ ).annotate(yyyy_mm=TruncYearMonth('responsetime')\ ).values('yyyy_mm'\ ).annotate(count=models.Count('id')\ ).values('yyyy_mm', 'count'\ ).order_by('%syyyy_mm' % self.time_sort) #print 'file_counts_by_month.query', file_counts_by_month.query sql_query = str(file_counts_by_month.query) formatted_records = [] # move from a queryset to a [] if count_pre_dv4_downloads: file_running_total = self.get_file_download_start_point_include_undated(**extra_filters) else: file_running_total = self.get_file_download_start_point(**extra_filters) for d in file_counts_by_month: fmt_rec = OrderedDict() fmt_rec['yyyy_mm'] = d['yyyy_mm'].strftime('%Y-%m') fmt_rec['count'] = d['count'] file_running_total += d['count'] fmt_rec['running_total'] = file_running_total # d['month_year'] = d['yyyy_mm'].strftime('%Y-%m') # Add year and month numbers fmt_rec['year_num'] = d['yyyy_mm'].year fmt_rec['month_num'] = d['yyyy_mm'].month # Add month name month_name_found, month_name_short = get_month_name_abbreviation( d['yyyy_mm'].month) if month_name_found: assume_month_name_found, fmt_rec['month_name'] = get_month_name(d['yyyy_mm'].month) fmt_rec['month_name_short'] = month_name_short else: # Log it!!!!!! pass formatted_records.append(fmt_rec) data_dict = OrderedDict() data_dict['total_downloads'] = file_running_total data_dict['record_count'] = len(formatted_records) data_dict['records'] = formatted_records return StatsResult.build_success_result(data_dict, sql_query)
def get_file_count_by_month(self, date_param=DVOBJECT_CREATEDATE_ATTR, **extra_filters): """ File counts by month """ # Was an error found earlier? # if self.was_error_found(): return self.get_error_msg_return() # ----------------------------------- # (1) Build query filters # ----------------------------------- # Exclude records where dates are null # - e.g. a record may not have a publication date if date_param == DVOBJECT_CREATEDATE_ATTR: exclude_params = {} else: exclude_params = { '%s__isnull' % date_param : True} # Retrieve the date parameters # filter_params = self.get_date_filter_params() # Add extra filters from kwargs # if extra_filters: for k, v in extra_filters.items(): filter_params[k] = v # ----------------------------------- # (2) Construct query # ----------------------------------- # add exclude filters date filters # file_counts_by_month = Datafile.objects.select_related('dvobject'\ ).exclude(**exclude_params\ ).filter(**filter_params) # annotate query adding "month_year" and "cnt" # file_counts_by_month = file_counts_by_month.annotate(\ yyyy_mm=TruncYearMonth('%s' % date_param)\ ).values('yyyy_mm'\ ).annotate(count=models.Count('dvobject_id')\ ).annotate(bytes=models.Sum('filesize')\ ).values('yyyy_mm', 'count', 'bytes'\ ).order_by('%syyyy_mm' % self.time_sort) sql_query = str(file_counts_by_month.query) # ----------------------------------- # (3) Format results # ----------------------------------- running_total = self.get_file_count_start_point(**extra_filters) # hold the running total count total_bytes = 0 formatted_records = [] # move from a queryset to a [] for d in file_counts_by_month: fmt_rec = OrderedDict() fmt_rec['yyyy_mm'] = d['yyyy_mm'].strftime('%Y-%m') fmt_rec['count'] = d['count'] fmt_rec['bytes'] = d['bytes'] fmt_rec['bytes_str'] = comma_sep_number(d['bytes']) total_bytes += d['bytes'] # running total running_total += d['count'] fmt_rec['running_total'] = running_total # d['month_year'] = d['yyyy_mm'].strftime('%Y-%m') # Add year and month numbers fmt_rec['year_num'] = d['yyyy_mm'].year fmt_rec['month_num'] = d['yyyy_mm'].month # Add month name month_name_found, month_name_short = get_month_name_abbreviation(d['yyyy_mm'].month) if month_name_found: assume_month_name_found, fmt_rec['month_name'] = get_month_name(d['yyyy_mm'].month) fmt_rec['month_name_short'] = month_name_short else: # Log it!!!!!! pass # Add formatted record formatted_records.append(fmt_rec) data_dict = OrderedDict() data_dict['record_count'] = len(formatted_records) data_dict['records'] = formatted_records data_dict['total_bytes'] = total_bytes data_dict['total_bytes_str'] = comma_sep_number(total_bytes) return StatsResult.build_success_result(data_dict, sql_query)
def get_dataverse_affiliation_counts(self, **extra_filters): """ Return Dataverse counts by affiliation Returns: dv_counts_by_affiliation": [ { "affiliation": "University of Oxford", "affiliation_count": 2, "total_count": 191, "percent_string": "1.0%" }, { "affiliation": "University of Illinois", "affiliation_count": 1, "total_count": 191, "percent_string": "0.5%" } ... ] """ if self.was_error_found(): return self.get_error_msg_return() # Retrieve the date parameters # filter_params = self.get_date_filter_params(DVOBJECT_CREATEDATE_ATTR) if extra_filters: for k, v in extra_filters.items(): filter_params[k] = v dataverse_counts_by_affil = Dataverse.objects.select_related('dvobject'\ ).filter(**filter_params\ ).values('affiliation'\ ).order_by('affiliation'\ ).annotate(affiliation_count=models.Count('affiliation')\ ).order_by('-affiliation_count') # ----------------------------------- # Get SQL query string # ----------------------------------- sql_query = str(dataverse_counts_by_affil.query) # Count all dataverses # total_count = sum([ rec.get('affiliation_count', 0) for rec in dataverse_counts_by_affil ]) total_count = total_count + 0.0 print 'dataverse_counts_by_affil', dataverse_counts_by_affil # Format the records, adding 'total_count' and 'percent_string' to each one # formatted_records = [] for rec in dataverse_counts_by_affil: if rec.get('affiliation_count', 0) > 0: fmt_dict = OrderedDict() affil_str = rec.get('affiliation', None) if affil_str is not None: affil_str = affil_str.encode('utf-8') fmt_dict['affiliation'] = affil_str fmt_dict['affiliation_count'] = rec.get('affiliation_count', 0) if total_count > 0: float_percent = rec.get('affiliation_count', 0) / total_count fmt_dict['total_count'] = int(total_count) fmt_dict['percent_string'] = '{0:.1%}'.format( float_percent) else: fmt_dict['total_count'] = 0 fmt_dict['percent_string'] = '0%' formatted_records.append(fmt_dict) data_dict = OrderedDict() data_dict['record_count'] = len(formatted_records) data_dict['records'] = formatted_records return StatsResult.build_success_result(data_dict, sql_query)
def get_dataset_subject_counts(self, **extra_filters): """Dataset counts by subjet""" # Was an error found earlier? # if self.was_error_found(): return self.get_error_msg_return() # ----------------------------------- # (1) Build query filters # ----------------------------------- # Retrieve the date parameters # ----------------------------------- filter_params = self.get_date_filter_params() # ----------------------------------- # Add extra filters from kwargs # ----------------------------------- if extra_filters: for k, v in extra_filters.items(): filter_params[k] = v # ----------------------------- # Get the DatasetFieldType for subject # ----------------------------- search_attrs = dict(name='subject',\ required=True,\ metadatablock__name='citation') try: ds_field_type = DatasetFieldType.objects.get(**search_attrs) except DatasetFieldType.DoesNotExist: return False, 'DatasetFieldType for Citation title not found. (kwargs: %s)' % search_attrs # ----------------------------- # Retrieve Dataset ids by time and published/unpublished # ----------------------------- dataset_ids = Dataset.objects.select_related('dvobject'\ ).filter(**filter_params\ ).values_list('dvobject__id', flat=True) # ----------------------------- # Get latest DatasetVersion ids # ----------------------------- id_info_list = DatasetVersion.objects.filter(\ dataset__in=dataset_ids\ ).values('id', 'dataset_id', 'versionnumber', 'minorversionnumber'\ ).order_by('dataset_id', '-id', '-versionnumber', '-minorversionnumber') # ----------------------------- # Iterate through and get the DatasetVersion id # of the latest version # ----------------------------- dsv_ids = [] last_dataset_id = None for idx, info in enumerate(id_info_list): if idx == 0 or info['dataset_id'] != last_dataset_id: dsv_ids.append(info['id']) last_dataset_id = info['dataset_id'] # ----------------------------- # Get the DatasetField ids # ----------------------------- search_attrs2 = dict(datasetversion__id__in=dsv_ids,\ datasetfieldtype__id=ds_field_type.id) ds_field_ids = DatasetField.objects.select_related( 'datasetfieldtype').filter(**search_attrs2).values_list('id', flat=True) # ----------------------------- # Finally, get the ControlledVocabularyValues # ----------------------------- ds_values = DatasetFieldControlledVocabularyValue.objects.select_related('controlledvocabularyvalues'\ ).filter(datasetfield__in=ds_field_ids\ ).annotate(subject=F('controlledvocabularyvalues__strvalue') ).values('subject'\ ).annotate(cnt=models.Count('controlledvocabularyvalues__id')\ ).values('subject', 'cnt'\ ).order_by('-cnt') # ----------------------------- # Iterate through the vocab values, # process the totals, calculate percentage # ----------------------------- running_total = 0 formatted_records = [] # move from a queryset to a [] total_count = sum([rec['cnt'] for rec in ds_values]) + 0.00 for info in ds_values: rec = OrderedDict() rec['subject'] = info['subject'] # count rec['count'] = info['cnt'] rec['total_count'] = int(total_count) # percent float_percent = info['cnt'] / total_count rec['percent_string'] = '{0:.1%}'.format(float_percent) rec['percent_number'] = float("%.3f" % (float_percent)) # total count formatted_records.append(rec) data_dict = OrderedDict() data_dict['record_count'] = len(formatted_records) data_dict['records'] = formatted_records return StatsResult.build_success_result(data_dict)
def get_dataset_count_by_month(self, date_param=DVOBJECT_CREATEDATE_ATTR, **extra_filters): """ Return dataset counts by month """ # Was an error found earlier? # if self.was_error_found(): return self.get_error_msg_return() # ----------------------------------- # (1) Build query filters # ----------------------------------- # Exclude records where dates are null # - e.g. a record may not have a publication date if date_param == DVOBJECT_CREATEDATE_ATTR: exclude_params = {} else: exclude_params = {'%s__isnull' % date_param: True} # Retrieve the date parameters # filter_params = self.get_date_filter_params() # Add extra filters from kwargs # if extra_filters: for k, v in extra_filters.items(): filter_params[k] = v # ----------------------------------- # (2) Construct query # ----------------------------------- # add exclude filters date filters # ds_counts_by_month = Dataset.objects.select_related('dvobject'\ ).exclude(**exclude_params\ ).filter(**filter_params) # annotate query adding "month_year" and "cnt" # ds_counts_by_month = ds_counts_by_month.annotate(\ yyyy_mm=TruncYearMonth('%s' % date_param)\ ).values('yyyy_mm'\ ).annotate(count=models.Count('dvobject_id')\ ).values('yyyy_mm', 'count'\ ).order_by('%syyyy_mm' % self.time_sort) # store query string sql_query = str(ds_counts_by_month.query) # ----------------------------------- # (3) Format results # ----------------------------------- # hold the running total count running_total = self.get_dataset_count_start_point(**extra_filters) formatted_records = [] # move from a queryset to a [] for d in ds_counts_by_month: fmt_dict = OrderedDict() fmt_dict['yyyy_mm'] = d['yyyy_mm'].strftime('%Y-%m') fmt_dict['count'] = d['count'] # running total running_total += d['count'] fmt_dict['running_total'] = running_total # d['month_year'] = d['yyyy_mm'].strftime('%Y-%m') # Add year and month numbers fmt_dict['year_num'] = d['yyyy_mm'].year fmt_dict['month_num'] = d['yyyy_mm'].month # Add month name month_name_found, month_name_short = get_month_name_abbreviation( d['yyyy_mm'].month) if month_name_found: assume_month_name_found, fmt_dict[ 'month_name'] = get_month_name(d['yyyy_mm'].month) fmt_dict['month_name_short'] = month_name_short else: # Log it!!!!!! pass # Add formatted record formatted_records.append(fmt_dict) data_dict = OrderedDict() data_dict['record_count'] = len(formatted_records) data_dict['records'] = formatted_records return StatsResult.build_success_result(data_dict, sql_query)
def get_published_dataverses_without_content(self, **extra_filters): """For curation purposes: a list of all published dataverses that do not contain any datasets/content. A spreadsheet starting with the oldest dataverses is appreciated. Based on @sekmiller's SQL query""" # Was an error found earlier? # if self.was_error_found(): return self.get_error_msg_return() # ----------------------------------- # Retrieve the date parameters - distinguish by create date # ----------------------------------- filter_params = self.get_date_filter_params() filter_params.update(self.get_is_published_filter_param()) # ----------------------------------- # Retrieve ids of Dataverses to ~exclude~ # ----------------------------------- # Get DvObject Ids of: # - Dataverses that contain Datasets # - Dataverses that have an owner # id_set1 = DvObject.objects.filter(\ Q(dtype=DTYPE_DATASET) |\ Q(dtype=DTYPE_DATAVERSE, owner__isnull=False) ).distinct('owner__id'\ ).values_list('owner__id', flat=True) # Get DvObject Ids of: # - Dataverses that link to datasets # id_set2 = DatasetLinkingDataverse.objects.distinct('linkingdataverse__id'\ ).values_list('linkingdataverse__id', flat=True) # Get DvObject Ids of: # - Dataverses that link to Dataverses # id_set3 = DataverseLinkingDataverse.objects.distinct('dataverse__id'\ ).values_list('dataverse__id', flat=True) # Combine the ids into a list # dv_ids_to_exclude = set( list(id_set1) + list(id_set2) + list((id_set3))) # Retrieve published Dataverses that aren't in the list above # dv_info_list = Dataverse.objects.select_related('dvobject'\ ).exclude(dvobject__id__in=dv_ids_to_exclude\ ).filter(**filter_params\ ).order_by(DVOBJECT_CREATEDATE_ATTR\ ).annotate(dv_id=F('dvobject__id'),\ create_date=F(DVOBJECT_CREATEDATE_ATTR),\ pub_date=F('dvobject__publicationdate')\ ).values('dv_id', 'name', 'alias'\ , 'create_date', 'pub_date'\ , 'affiliation'\ ).order_by('create_date', 'name') sql_query = str(q.query) records = [] for dv_info in dv_info_list: single_rec = OrderedDict() single_rec['id'] = dv_info['dv_id'] single_rec['name'] = dv_info['name'] single_rec['alias'] = dv_info['alias'] single_rec['url'] = '%s/dataverse/%s' % ( settings.DATAVERSE_INSTALLATION_URL, dv_info['alias']) #single_rec['description'] = dv_info['description'] single_rec['affiliation'] = dv_info['affiliation'] single_rec['publication_date'] = dv_info['pub_date'].strftime( TIMESTAMP_MASK) single_rec['create_date'] = dv_info['create_date'].strftime( TIMESTAMP_MASK) records.append(single_rec) data_dict = OrderedDict() data_dict['count'] = len(records) data_dict['records'] = records return StatsResult.build_success_result(data_dict, sql_query)
def get_file_counts_per_dataset_latest_versions(self, **extra_filters): """ Get binning stats for the number of files in each Dataset. For the counts, only use the LATEST DatasetVersion """ # Get the correct DatasetVersion ids as a filter parameter # latest_dsv_ids = self.get_dataset_version_ids(**extra_filters) filter_params = dict(datasetversion__id__in=latest_dsv_ids) # Make query # ds_version_counts = FileMetadata.objects.filter(**filter_params\ ).annotate(dsv_id=F('datasetversion__id'),\ ).values('dsv_id',\ ).annotate(cnt=models.Count('datafile__id')\ ).values('dsv_id', 'cnt'\ ).order_by('-cnt') # Convert to Dataframe # df = pd.DataFrame(list(ds_version_counts), columns=['dsv_id', 'cnt']) # Get the list of bins # high_num = high_num = df['cnt'].max() + self.bin_size bins = self.get_bin_list(step=self.bin_size, low_num=0, high_num=high_num + self.bin_size) # Add a new column, assigning each file count to a bin # df['bin_label'] = pd.cut(df['cnt'], bins) # Count the occurrence of each bin # bin_count_series = pd.value_counts(df['bin_label']) # Make the Series into a new DataFrame # df_bins = pd.DataFrame(dict(bin=bin_count_series.index,\ count=bin_count_series.values)) # Add a sort key # (0, 20] -> 0 # (20, 30] -> 20 # etc df_bins['sort_key'] = df_bins['bin'].apply( lambda x: int(x[1:-1].split(',')[0])) df_bins['bin_start_inclusive'] = df_bins['sort_key'] df_bins['bin_end'] = df_bins['bin'].apply( lambda x: int(x[1:-1].split(',')[1])) # Add a formatted string # (0, 20] -> 0 to 20 # (20, 30] -> 20 to 30 # etc df_bins['bin_str'] = df_bins['bin'].apply( lambda x: x[1:-1].replace(', ', ' to ')) # Sort the bins # df_bins = df_bins.sort('sort_key') #msgt(df_bins) # If appropriate, skip empty bins, e.g. remove 0 counts # if self.skip_empty_bins: df_bins = df_bins.query('count != 0') #msg(df_bins) # Return as python dict # # bit expensive but want orderedDict formatted_records_json = df_bins.to_json(orient='records') formatted_records = json.loads(formatted_records_json, object_pairs_hook=OrderedDict) data_dict = OrderedDict() data_dict['record_count'] = len(formatted_records) data_dict['records'] = formatted_records return StatsResult.build_success_result(data_dict)
def get_dataset_size_counts(self, **extra_filters): """ Get binning stats for the byte size of each Dataset. """ # Get the correct DatasetVersion ids as a filter parameter # filter_params = {} if extra_filters: filter_params.update(extra_filters) # Make query # dataset_file_sizes = Datafile.objects.filter(**filter_params\ ).annotate(ds_id=F('dvobject__owner__id'),\ ).values('ds_id',\ ).annotate(cnt=models.Count('dvobject__id')\ , ds_size=Sum('filesize') ).values('ds_id', 'cnt', 'ds_size'\ ).order_by('ds_size') #total_bytes_used_result = Datafile.objects.filter(**filter_params\ # ).aggregate(ds_size=Sum('filesize')) # Convert to Dataframe # df = pd.DataFrame(list(dataset_file_sizes), columns = ['ds_id', 'cnt', 'ds_size']) #total_dataset_count = len(df.index) # * includes rows with missing values total_bytes_used = df['ds_size'].sum() # Get the list of bins # high_num = df['ds_size'].max() + self.bin_size_bytes bins = self.get_bin_list(step=self.bin_size_bytes, low_num=0, high_num=high_num+self.bin_size_bytes) # Add a new column, assigning each file count to a bin # df['bin_label'] = pd.cut(df['ds_size'], bins) # Count the occurrence of each bin # bin_count_series = pd.value_counts(df['bin_label']) # Make the Series into a new DataFrame # df_bins = pd.DataFrame(dict(bin=bin_count_series.index,\ count=bin_count_series.values)) total_dataset_count = df_bins['count'].sum() # Add a sort key # (0, 20] -> 0 # (20, 30] -> 20 # etc df_bins['sort_key'] = df_bins['bin'].apply(lambda x: int(x[1:-1].split(',')[0])) if total_dataset_count > 0: df_bins['percentage_of_datasets'] = df_bins['count'].apply(lambda x: "{0:.4f}%".format(100 * x/float(total_dataset_count))) #100*x/float(x.sum()) df_bins['bin_start_inclusive'] = df_bins['sort_key'] df_bins['bin_start_inclusive_commas'] = df_bins['bin_start_inclusive'].apply(lambda x: comma_sep_number(x)) df_bins['bin_start_inclusive_abbrev'] = df_bins['bin_start_inclusive'].apply(lambda x: sizeof_fmt(x)) df_bins['bin_end'] = df_bins['bin'].apply(lambda x: int(x[1:-1].split(',')[1])) df_bins['bin_end_commas'] = df_bins['bin_end'].apply(lambda x: comma_sep_number(x)) df_bins['bin_end_abbrev'] = df_bins['bin_end'].apply(lambda x: sizeof_fmt(x)) df_bins['bin_str'] = df_bins['bin_start_inclusive_abbrev'].str.cat(df_bins['bin_end_abbrev'].values.astype(str), sep=' to ') # Sort the bins # df_bins = df_bins.sort('sort_key') #msgt(df_bins) # If appropriate, skip empty bins, e.g. remove 0 counts # if self.skip_empty_bins: df_bins = df_bins.query('count != 0') #msg(df_bins) # Return as python dict # # bit expensive but want orderedDict formatted_records_json = df_bins.to_json(orient='records') formatted_records = json.loads(formatted_records_json, object_pairs_hook=OrderedDict) data_dict = OrderedDict() data_dict['record_count'] = len(formatted_records) data_dict['dataset_count'] = total_dataset_count data_dict['total_bytes_used'] = total_bytes_used data_dict['total_bytes_used_comma'] = comma_sep_number(int(total_bytes_used)) data_dict['total_bytes_used_abbrev'] = sizeof_fmt(total_bytes_used) data_dict['records'] = formatted_records return StatsResult.build_success_result(data_dict)
def get_dataverse_counts_by_type(self, exclude_uncategorized=True, **extra_filters): """ Return dataverse counts by 'dataversetype' Optional if a dataverse is uncategorized: - Specifying 'uncategorized_replacement_name' will set "UNCATEGORIZED" to another string Returns: { "dv_counts_by_type": [ { "dataversetype": "RESEARCH_PROJECTS", "type_count": 85, "total_count": 356, "percent_string": "23.9%" }, { "dataversetype": "TEACHING_COURSES", "type_count": 10, "total_count": 356, "percent_string": "2.8%" } ... etc ] } """ if self.was_error_found(): return self.get_error_msg_return() # Retrieve the date parameters # filter_params = self.get_date_filter_params(DVOBJECT_CREATEDATE_ATTR) # Add extra filters if extra_filters: for k, v in extra_filters.items(): filter_params[k] = v if exclude_uncategorized: exclude_params = dict(dataversetype=DATAVERSE_TYPE_UNCATEGORIZED) else: exclude_params = {} dataverse_counts_by_type = Dataverse.objects.select_related('dvobject'\ ).filter(**filter_params\ ).exclude(**exclude_params\ ).values('dataversetype'\ ).order_by('dataversetype'\ ).annotate(type_count=models.Count('dataversetype')\ ).order_by('-type_count') # ----------------------------------- # Get SQL query string # ----------------------------------- sql_query = str(dataverse_counts_by_type.query) # Count all dataverses # total_count = sum( [rec.get('type_count', 0) for rec in dataverse_counts_by_type]) total_count = total_count + 0.0 # Format the records, adding 'total_count' and 'percent_string' to each one # formatted_records = [] for rec in dataverse_counts_by_type: fmt_dict = OrderedDict() fmt_dict['dataversetype'] = rec['dataversetype'] fmt_dict['dataversetype_label'] = rec['dataversetype'].replace( '_', ' ') fmt_dict['type_count'] = rec.get('type_count', 0) if total_count > 0: float_percent = rec.get('type_count', 0) / total_count fmt_dict['total_count'] = int(total_count) fmt_dict['percent_string'] = '{0:.1%}'.format(float_percent) else: fmt_dict['total_count'] = 0 fmt_dict['percent_string'] = '0%' formatted_records.append(fmt_dict) data_dict = OrderedDict() data_dict['record_count'] = len(formatted_records) data_dict['records'] = formatted_records return StatsResult.build_success_result(data_dict, sql_query)