def view_dataverse_list(request, output_format='xlsx', **kwargs): filter_params = {} published_only = kwargs.get('published_only', True) if published_only: filter_params.update(query_helper.get_is_published_filter_param()) vals = [ 'id', 'name', 'alias', 'dataversetype', 'createdate', 'publicationdate' ] dlist = Dataverse.objects.select_related('dvobject'\ ).filter(**filter_params ).annotate(id=F('dvobject__id'), createdate=F('dvobject__createdate'), publicationdate=F('dvobject__publicationdate'), ).values(*vals\ ).order_by('alias') df = pd.DataFrame(list(dlist), columns=vals) df['dataverse_url'] = df['alias'].apply( lambda x: 'https://dataverse.harvard.edu/dataverse/%s' % x) vals.append('dataverse_url') if output_format == 'xlsx': excel_string_io = StringIO.StringIO() pd_writer = pd.ExcelWriter(excel_string_io, engine='xlsxwriter') df.to_excel(pd_writer, index=False, sheet_name='metrics', columns=vals) pd_writer.save() excel_string_io.seek(0) workbook = excel_string_io.getvalue() if workbook is None: # Ah, make a better error return HttpResponse( 'Sorry! An error occurred trying to create an Excel spreadsheet.' ) xlsx_fname = 'dataverses_%s.xlsx' % get_timestamp_for_filename() response = HttpResponse(workbook,\ content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') response[ 'Content-Disposition'] = 'attachment; filename=%s' % xlsx_fname return response return HttpResponse('Sorry. This format is not recognized: %s' % output_format)
def view_single_dataverse_by_alias(request, alias): """JSON repr. of Dataverse. Published Dataverses only""" query_params = dict(alias=alias) query_params.update(query_helper.get_is_published_filter_param()) try: dv = Dataverse.objects.select_related('dvobject').get(**query_params) except Dataverse.DoesNotExist: raise Http404 return _view_single_dataverse(request, dv)
def view_dataverse_list(request, output_format='xlsx', **kwargs): filter_params = {} published_only = kwargs.get('published_only', True) if published_only: filter_params.update(query_helper.get_is_published_filter_param()) vals = ['id', 'name', 'alias', 'dataversetype', 'createdate', 'publicationdate' ] dlist = Dataverse.objects.select_related('dvobject'\ ).filter(**filter_params ).annotate(id=F('dvobject__id'), createdate=F('dvobject__createdate'), publicationdate=F('dvobject__publicationdate'), ).values(*vals\ ).order_by('alias') df = pd.DataFrame(list(dlist), columns=vals) df['dataverse_url'] = df['alias'].apply(lambda x: 'https://dataverse.harvard.edu/dataverse/%s' % x) vals.append('dataverse_url') if output_format == 'xlsx': excel_string_io = StringIO.StringIO() pd_writer = pd.ExcelWriter(excel_string_io, engine='xlsxwriter') df.to_excel(pd_writer, index=False, sheet_name='metrics', columns=vals) pd_writer.save() excel_string_io.seek(0) workbook = excel_string_io.getvalue() if workbook is None: # Ah, make a better error return HttpResponse('Sorry! An error occurred trying to create an Excel spreadsheet.') xlsx_fname = 'dataverses_%s.xlsx' % get_timestamp_for_filename() response = HttpResponse(workbook,\ content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') response['Content-Disposition'] = 'attachment; filename=%s' % xlsx_fname return response return HttpResponse('Sorry. This format is not recognized: %s' % output_format)
def view_file_content_types(request): """JSON list of content types""" is_pretty = False as_excel = False if request.GET.get('as_excel') is not None: as_excel = True elif request.GET.get('pretty') is not None: is_pretty = True is_published_param = query_helper.get_is_published_filter_param() content_types = Datafile.objects.select_related('dvobject'\ ).filter(**is_published_param\ ).values_list('contenttype', flat=True\ ).distinct().order_by('contenttype') d = OrderedDict() d['content_types'] = list(content_types) if as_excel: if len(d) == 0: return HttpResponse('nothing found') # https://stackoverflow.com/questions/35267585/django-pandas-to-http-response-download-file df = pd.read_json(json.dumps(d)) excel_file = StringIO.StringIO() xlwriter = pd.ExcelWriter(excel_file, engine='xlsxwriter') df.to_excel(xlwriter, 'content_types', index=False) xlwriter.save() xlwriter.close() excel_file.seek(0) # set the mime type so that the browser knows what to do with the file response = HttpResponse(\ excel_file.read(), content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') # set the file name in the Content-Disposition header response['Content-Disposition'] = 'attachment; filename=dv_content_types.xlsx' return response d['description'] = 'Content types of published Dataverse files' if is_pretty: json_str = '<pre>%s</pre>' % json.dumps(d, indent=4) return HttpResponse(json_str) else: json_str = json.dumps(d) return HttpResponse(json_str, content_type="application/json")
def get_latest_dataset_version(dataset_id): """Given a dataset id, retrieve the latest *published* DatasetVersion""" dataset = Dataset.objects.select_related('dvobject'\ ).filter(dvobject__id=dataset_id\ ).filter(**query_helper.get_is_published_filter_param()\ ).first() if dataset is None: return None # Get the latest version dataset_version = DatasetVersion.objects\ .select_related('dataset')\ .filter(dataset=dataset,\ versionstate=VERSION_STATE_RELEASED)\ .order_by('-id').first() return dataset_version
def get_table_rows(datafile_id): if not datafile_id: err_msg = "No file id specified" return False, err_msg, 400 is_published_param = query_helper.get_is_published_filter_param() datafile = Datafile.objects.select_related('dvobject' ).filter(**is_published_param ).filter(dvobject__id=datafile_id).first() if datafile is None: err_msg = "No published file found with id: %s" % datafile_id return False, err_msg, 404 file_access_url = DatafileUtil.get_file_access_url(datafile_id) temp_filepath, file_ext = temp_file_helper.download_file(file_access_url) if temp_filepath is None: err_msg = "Failed to download file" return False, err_msg, 400 previewer = TabularPreviewer(temp_filepath, **dict(file_ext=file_ext)) if previewer.has_error(): return False, previewer.error_message, 400 data_rows = previewer.get_data_rows() if data_rows is None: return False, previewer.error_message, 500 # We have the rows, delete the downloaded file # In future, cache or save preview rows to db, etc. # temp_file_helper.make_sure_file_deleted(temp_filepath) return True, data_rows, 200
def get_table_rows(datafile_id): if not datafile_id: err_msg = "No file id specified" return False, err_msg, 400 is_published_param = query_helper.get_is_published_filter_param() datafile = Datafile.objects.select_related('dvobject').filter( **is_published_param).filter(dvobject__id=datafile_id).first() if datafile is None: err_msg = "No published file found with id: %s" % datafile_id return False, err_msg, 404 file_access_url = DatafileUtil.get_file_access_url(datafile_id) temp_filepath, file_ext = temp_file_helper.download_file(file_access_url) if temp_filepath is None: err_msg = "Failed to download file" return False, err_msg, 400 previewer = TabularPreviewer(temp_filepath, **dict(file_ext=file_ext)) if previewer.has_error(): return False, previewer.error_message, 400 data_rows = previewer.get_data_rows() if data_rows is None: return False, previewer.error_message, 500 # We have the rows, delete the downloaded file # In future, cache or save preview rows to db, etc. # temp_file_helper.make_sure_file_deleted(temp_filepath) return True, data_rows, 200
def make_json_files(self): # Set publication status # filters = {} if self.published_only: filters.update(query_helper.get_is_published_filter_param()) # Query for dataset ids # ds_id_query = Dataset.objects.filter(**filters\ ).annotate(ds_id=F('dvobject__id')\ ).values_list('ds_id', flat=True\ ).order_by('ds_id') # Iterate through dataset ids # #start_time = datetime.now() start_time = int(time.time()) # epoch seconds cnt = 0 no_versions_found_list = [45900] for ds_id in ds_id_query: cnt += 1 msgt('(%d) Checking dataset id %s' % (cnt, ds_id)) if ds_id < self.dataset_start_id: msg('skipping...(start at dataset id: %d)' % self.dataset_start_id) continue # Create file name # fname = 'ds_%s.json' % (str(ds_id).zfill(8)) full_fname = join(OUTPUT_DIR, fname) # Should we overwrite the existing file? # if isfile(full_fname) and not self.overwrite_existing_files: msg('skipping...file already exists') continue dataset_version = get_latest_dataset_version(ds_id) if dataset_version is None: msg("Could not find dataset_version!") no_versions_found_list.append(ds_id) continue dataset_as_json = DatasetSerializer(dataset_version).as_json() open(full_fname, 'w').write(json.dumps(dataset_as_json, indent=4)) msg('File written: %s' % full_fname) if cnt % 500 == 0: self.show_elapsed_time(start_time) #if cnt > 10: # self.show_elapsed_time(start_time) # break self.show_elapsed_time(start_time) print 'no_versions_found_list: %s' % no_versions_found_list
def get_is_published_filter_param(self, dvobject_var_name='dvobject'): """ Check if the dvobject has a publication date--which indicates that it has been published """ return query_helper.get_is_published_filter_param(dvobject_var_name)
def view_file_list_by_type(request): """Give a list of published files based on the content type. e.g. ?contenttype=text/tab-separated-values """ is_pretty = False as_excel = False if request.GET.get('as_excel') is not None: as_excel = True elif request.GET.get('pretty') is not None: is_pretty = True contenttype = request.GET.get('contenttype', CONTENT_TYPE_TABULAR) query_params = dict(contenttype=contenttype, filesize__gt=0, checksumvalue__isnull=False, restricted=False, ingeststatus=INGEST_STATUS_NONE) query_params.update(query_helper.get_is_published_filter_param()) dfiles = Datafile.objects.select_related('dvobject', 'dvobject__owner'\ ).filter(**query_params\ ).order_by('contenttype') flist = [Datafile.to_json(df)\ for df in dfiles\ if df is not None and df.id] d = OrderedDict() d['file_list'] = flist if as_excel: # https://stackoverflow.com/questions/35267585/django-pandas-to-http-response-download-file if len(flist) == 0: return HttpResponse('no published files for contenttype: %s' % contenttype) df = pd.DataFrame(flist, columns=flist[0].keys()) excel_file = StringIO.StringIO() xlwriter = pd.ExcelWriter(excel_file, engine='xlsxwriter') df.to_excel(xlwriter, 'file_list', index=False) xlwriter.save() xlwriter.close() excel_file.seek(0) # set the mime type so that the browser knows what to do with the file response = HttpResponse(\ excel_file.read(), content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') # set the file name in the Content-Disposition header response['Content-Disposition'] = 'attachment; filename=dv_file_list.xlsx' return response d['description'] = 'Files with content type %s. Count: %d' % (contenttype, len(flist)) if is_pretty: json_str = '<pre>%s</pre>' % json.dumps(d, indent=4) return HttpResponse(json_str) else: json_str = json.dumps(d) return HttpResponse(json_str, content_type="application/json")
def make_json_files(self): # Set publication status # filters = {} if self.published_only: filters.update(query_helper.get_is_published_filter_param()) # Query for dataset ids # ds_id_query = Dataset.objects.filter(**filters\ ).annotate(ds_id=F('dvobject__id')\ ).values_list('ds_id', flat=True\ ).order_by('ds_id') # Iterate through dataset ids # #start_time = datetime.now() start_time = int(time.time()) # epoch seconds cnt = 0 no_versions_found_list = [45900] for ds_id in ds_id_query: cnt += 1 msgt('(%d) Checking dataset id %s' % (cnt, ds_id)) if ds_id < self.dataset_start_id: msg('skipping...(start at dataset id: %d)' % self.dataset_start_id) continue # Create file name # fname = 'ds_%s.json' % (str(ds_id).zfill(8)) full_fname = join(OUTPUT_DIR, fname) # Should we overwrite the existing file? # if isfile(full_fname) and not self.overwrite_existing_files: msg('skipping...file already exists') continue dataset_version = get_latest_dataset_version(ds_id) if dataset_version is None: msg("Could not find dataset_version!") no_versions_found_list.append(ds_id) continue dataset_as_json = DatasetSerializer(dataset_version).as_json() open(full_fname, 'w').write(json.dumps(dataset_as_json, indent=4)) msg('File written: %s' % full_fname) if cnt % 500 == 0: self.show_elapsed_time(start_time) #if cnt > 10: # self.show_elapsed_time(start_time) # break self.show_elapsed_time(start_time) print 'no_versions_found_list: %s' % no_versions_found_list