def download_file(url_to_file): """Download a Dataverse file and return the filename""" """ import re d = r.headers['content-disposition'] fname = re.findall("filename=(.+)", d) """ file_handle, filepath = tempfile.mkstemp() msgt('download file: %s' % url_to_file) r = requests.get(url_to_file, stream=True) if r.status_code != 200: msg('bad status: %s' % r.status_code) if isfile(filepath): make_sure_file_deleted(filepath) return None, None file_ext = None content_dict = r.headers['content-disposition'] #print 'content_dict', content_dict #fname = re.findall("filename=(.+)", content_dict) fname = format_file_name(content_dict) if fname: file_ext = fname.split('.')[-1].lower() print 'file_ext', file_ext with os.fdopen(file_handle, 'wb') as tmp: for chunk in r.iter_content(chunk_size=1024): if chunk: tmp.write(chunk) msg('File downloaded: %s' % filepath) return filepath, file_ext
def compare_dicts(self, section, new_dict, old_dict, **kwargs): """Compare two dicts, noting whether a key/value was Removed, Added, or Modified Note: Attempts to preserve key order--dict is usually an OrderedDict """ # optional: get keys to skip skip_list = kwargs.get('skip_list', []) old_dict_keys = [key for key, val in old_dict.items()] new_dict_keys = [key for key, val in new_dict.items()] #old_dict_keys = set(old_dict.keys()) #new_dict_keys = set(new_dict.keys()) # ------------------------------ # added attributes # ------------------------------ added = [key for key in new_dict_keys if key not in old_dict_keys] #new_dict_keys - old_dict_keys for added_key in added: if added_key not in skip_list: self.record_diff_desc_added(\ section, added_key, new_dict[added_key]) # ------------------------------ # Removed attributes # ------------------------------ removed = [key for key in old_dict_keys if key not in new_dict_keys] for removed_key in removed: if removed_key not in skip_list: self.record_diff_desc_removed(\ section, removed_key, old_dict[removed_key]) # ------------------------------ # Modified attributes # ------------------------------ intersect_keys = [key for key in new_dict_keys\ if key not in removed and key not in added] mod_keys = [shared_key for shared_key in intersect_keys \ if old_dict[shared_key] != new_dict[shared_key]] for mod_key in mod_keys: msg('...> %s %s' % (mod_key, type(old_dict[mod_key]))) # Is the value another dict? # if mod_key not in skip_list: self.compare_items(\ section, mod_key, new_dict[mod_key], old_dict[mod_key])
def compare_dicts(self, section, new_dict, old_dict, **kwargs): """Compare two dicts, noting whether a key/value was Removed, Added, or Modified Note: Attempts to preserve key order--dict is usually an OrderedDict """ # optional: get keys to skip skip_list = kwargs.get('skip_list', []) old_dict_keys = [key for key, val in old_dict.items()] new_dict_keys = [key for key, val in new_dict.items()] #old_dict_keys = set(old_dict.keys()) #new_dict_keys = set(new_dict.keys()) # ------------------------------ # added attributes # ------------------------------ added = [key for key in new_dict_keys if key not in old_dict_keys] #new_dict_keys - old_dict_keys for added_key in added: if added_key not in skip_list: self.record_diff_desc_added(\ section, added_key, new_dict[added_key]) # ------------------------------ # Removed attributes # ------------------------------ removed = [key for key in old_dict_keys if key not in new_dict_keys] for removed_key in removed: if removed_key not in skip_list: self.record_diff_desc_removed(\ section, removed_key, old_dict[removed_key]) # ------------------------------ # Modified attributes # ------------------------------ intersect_keys = [key for key in new_dict_keys\ if key not in removed and key not in added] mod_keys = [shared_key for shared_key in intersect_keys \ if old_dict[shared_key] != new_dict[shared_key]] for mod_key in mod_keys: msg('...> %s %s' % (mod_key, type(old_dict[mod_key]))) # Is the value another dict? # if mod_key not in skip_list: self.compare_items(\ section, mod_key, new_dict[mod_key], old_dict[mod_key])
def write_files_to_mongo(self, **kwargs): """Write the saved dataset files to Mongo""" client = MongoClient() db = client.dataverse_database collection = db.datasets # look at kwargs # dataset_start_id = kwargs.get('dataset_start_id', 0) delete_all = kwargs.get('delete_all', False) # If appropriate, Delete existing records # if delete_all: msgt('Deleting current records') result = collection.delete_many({}) msg('result.deleted_count: %s' % result.deleted_count) return fnames = os.listdir(self.output_dir) fnames = [ x for x in fnames if x.endswith('.json') and x.startswith('ds_') ] fnames.sort() start_time = int(time.time()) # epoch seconds cnt = 0 for fname in fnames: cnt += 1 ds_id = int(fname.split('.')[0].split('_')[1]) msgt('(%d) process dataset %s (%s)' % (cnt, ds_id, fname)) if ds_id < dataset_start_id: msg('skipping it') continue content = open(join(self.output_dir, fname), 'r').read() content = update_json_text(content) content_doc = json.loads(content, object_pairs_hook=OrderedDict) content_doc['_id'] = ds_id content_doc['dtype'] = 'dataset' #doc_id = collection.insert_one(content_doc).inserted_id #doc_id = collection.save(content_doc) #.inserted_id doc_id = collection.save(content_doc) if cnt % 500 == 0: self.show_elapsed_time(start_time) self.show_elapsed_time(start_time)
def write_files_to_mongo(self, **kwargs): """Write the saved dataset files to Mongo""" client = MongoClient() db = client.dataverse_database collection = db.datasets # look at kwargs # dataset_start_id = kwargs.get('dataset_start_id', 0) delete_all = kwargs.get('delete_all', False) # If appropriate, Delete existing records # if delete_all: msgt('Deleting current records') result = collection.delete_many({}) msg('result.deleted_count: %s' % result.deleted_count) return fnames = os.listdir(self.output_dir) fnames = [x for x in fnames if x.endswith('.json') and x.startswith('ds_')] fnames.sort() start_time = int(time.time()) # epoch seconds cnt = 0 for fname in fnames: cnt += 1 ds_id = int(fname.split('.')[0].split('_')[1]) msgt('(%d) process dataset %s (%s)' % (cnt, ds_id, fname)) if ds_id < dataset_start_id: msg('skipping it') continue content = open(join(self.output_dir, fname), 'r').read() content = update_json_text(content) content_doc = json.loads(content, object_pairs_hook=OrderedDict) content_doc['_id'] = ds_id content_doc['dtype'] = 'dataset' #doc_id = collection.insert_one(content_doc).inserted_id #doc_id = collection.save(content_doc) #.inserted_id doc_id = collection.save(content_doc) if cnt % 500 == 0: self.show_elapsed_time(start_time) self.show_elapsed_time(start_time)
def run_comparison(self): """Compare the two JSON datasets""" msgt(self.run_comparison.__doc__) # Run a quick check to see if the dicts are the same. # if cmp(self.old_ds, self.new_ds) == 0: msg('No differences!') return new_files_list = self.new_ds.pop('files', []) old_files_list = self.old_ds.pop('files', []) #print 'new_files_list', new_files_list self.compare_dicts(\ '', self.new_ds, self.old_ds) self.compare_file_lists(\ new_files_list, old_files_list)
def download_file(url_to_file): """Download a Dataverse file and return the filename""" file_handle, filepath = tempfile.mkstemp() msgt('download file: %s' % url_to_file) r = requests.get(url_to_file, stream=True) if r.status_code != 200: msg('bad status: %s' % r.status_code) if isfile(filepath): make_sure_file_deleted(filepath) return None with os.fdopen(file_handle, 'wb') as tmp: for chunk in r.iter_content(chunk_size=1024): if chunk: tmp.write(chunk) msg('File downloaded: %s' % filepath) return filepath
def run_comparison(self): """Compare the two JSON datasets""" msgt(self.run_comparison.__doc__) # Run a quick check to see if the dicts are the same. # if cmp(self.old_ds, self.new_ds) == 0: msg('No differences!') return new_files_list = self.new_ds.pop('files', []) old_files_list = self.old_ds.pop('files', []) #print 'new_files_list', new_files_list self.compare_dicts(\ '', self.new_ds, self.old_ds) self.compare_file_lists(\ new_files_list, old_files_list)
def download_file(url_to_file): """Download a Dataverse file and return the filename""" """ import re d = r.headers['content-disposition'] fname = re.findall("filename=(.+)", d) """ file_handle, filepath = tempfile.mkstemp() msgt('download file: %s' % url_to_file) r = requests.get(url_to_file, stream=True) if r.status_code != 200: msg('bad status: %s' % r.status_code) if isfile(filepath): make_sure_file_deleted(filepath) return None, None file_ext = None content_dict = r.headers['content-disposition'] #print 'content_dict', content_dict #fname = re.findall("filename=(.+)", content_dict) fname = format_file_name(content_dict) if fname: file_ext = fname.split('.')[-1].lower() print 'file_ext', file_ext with os.fdopen(file_handle, 'wb') as tmp: for chunk in r.iter_content(chunk_size=1024): if chunk: tmp.write(chunk) msg('File downloaded: %s' % filepath) return filepath, file_ext
def show(self, show_section=True): """print info""" if show_section: msgt('%s: [%s] %s' % (self.section, self.attr_name, self.note)) msg('attribute: %s' % self.attr_name) msg('\nnew: %s' % self.new_val) msg('\nold: %s' % self.old_val) dashes()
def show(self, show_section=True): """print info""" if show_section: msgt('%s: [%s] %s' % (self.section, self.attr_name, self.note)) msg('attribute: %s' % self.attr_name) msg('\nnew: %s' % self.new_val) msg('\nold: %s' % self.old_val) dashes()
def get_count_broken_notifications(): """ Query each object type and make sure notifications aren't broken Example map { 'DvObject': [1], 'Dataverse': [2], 'Dataset': [14, 11], 'DatasetVersion': [13, 12, 7], 'DataFile': [9] } """ broken_cnt = 0 user_ids = [] for model_name, type_id_list in get_dv_object_to_object_id_map().items( ): # Get a list of object ids for this model type # that were not emailed--e.g. should show up # on the notifications pages # msgt('check: %s %s' % (model_name, type_id_list)) model_user_id_list = UserNotification.objects.select_related('user'\ ).filter(\ object_type__in=type_id_list, ).values_list('objectid', 'user__id') model_id_list = [x[0] for x in model_user_id_list] user_ids += [x[1] for x in model_user_id_list] msg('model_id_list len: %s' % len(model_id_list)) if len(model_id_list) == 0: continue # Used for later "bad notice" counts notice_counter = Counter(model_id_list) msg('notice_counter len: %s' % len(notice_counter)) unique_id_list = list(set(model_id_list)) msg('unique_id_list len: %s' % len(unique_id_list)) # Need to upgrade apps files and not use this method model_class = eval(model_name) if model_name in ['DvObject', 'DatasetVersion', 'FileMetadata']: existing_ids = model_class.objects.filter(id__in=unique_id_list\ ).values_list('id', flat=True\ ).distinct() else: existing_ids = model_class.objects.select_related('dvobject'\ ).filter(dvobject__id__in=unique_id_list\ ).values_list('dvobject__id', flat=True\ ).distinct() msg('existing_ids len: %s' % len(existing_ids)) if len(unique_id_list) == len(existing_ids): # Looks good! continue missing_ids = list(set(unique_id_list) - set(existing_ids)) for missing_id in missing_ids: broken_cnt += notice_counter.get(missing_id, 0) unique_user_ids = len(set(user_ids)) return (broken_cnt, unique_user_ids)
def make_json_files(self): # Set publication status # filters = {} if self.published_only: filters.update(query_helper.get_is_published_filter_param()) # Query for dataset ids # ds_id_query = Dataset.objects.filter(**filters\ ).annotate(ds_id=F('dvobject__id')\ ).values_list('ds_id', flat=True\ ).order_by('ds_id') # Iterate through dataset ids # #start_time = datetime.now() start_time = int(time.time()) # epoch seconds cnt = 0 no_versions_found_list = [45900] for ds_id in ds_id_query: cnt += 1 msgt('(%d) Checking dataset id %s' % (cnt, ds_id)) if ds_id < self.dataset_start_id: msg('skipping...(start at dataset id: %d)' % self.dataset_start_id) continue # Create file name # fname = 'ds_%s.json' % (str(ds_id).zfill(8)) full_fname = join(OUTPUT_DIR, fname) # Should we overwrite the existing file? # if isfile(full_fname) and not self.overwrite_existing_files: msg('skipping...file already exists') continue dataset_version = get_latest_dataset_version(ds_id) if dataset_version is None: msg("Could not find dataset_version!") no_versions_found_list.append(ds_id) continue dataset_as_json = DatasetSerializer(dataset_version).as_json() open(full_fname, 'w').write(json.dumps(dataset_as_json, indent=4)) msg('File written: %s' % full_fname) if cnt % 500 == 0: self.show_elapsed_time(start_time) #if cnt > 10: # self.show_elapsed_time(start_time) # break self.show_elapsed_time(start_time) print 'no_versions_found_list: %s' % no_versions_found_list
def get_basic_stats(): cnt_read_notifications = UserNotification.objects.filter(\ readnotification=True, ).count() cnt_unread_notifications = UserNotification.objects.filter(\ readnotification=False, ).count() cnt_undated_notifications = UserNotification.objects.filter(\ senddate__isnull=True ).count() day_cnt_1 = 365 day_cnt_1_date = datetime.now() - timedelta(days=day_cnt_1) day_cnt_2 = 180 day_cnt_2_date = datetime.now() - timedelta(days=day_cnt_2) cnt_old_unread_notifications = UserNotification.objects.filter(\ readnotification=False, senddate__lt=day_cnt_1_date ).count() cnt_old_unread_notifications2 = UserNotification.objects.filter(\ readnotification=False, senddate__lt=day_cnt_2_date ).count() broken_cnt, impacted_users = NotificationStats.get_count_broken_notifications() msg('broken_cnt: %s' % broken_cnt) msg('impacted_users: %s' % impacted_users) file_stats = dict(\ cnt_broken_notifications=NamedStat(\ 'Broken Notifications / Impacted Users', broken_cnt, ('The notification refers to an object that' ' no longer exists. These notifications should' ' be deleted from the database. (May be' ' responsible for some users who receive an' ' error when clicking on the notifications' ' tab.)'), 'view_broken_notifications', **dict(stat2=impacted_users)), cnt_read_notifications=NamedStat(\ 'Read Notifications', cnt_read_notifications, ('Count of read notifications'), None), cnt_unread_notifications=NamedStat(\ 'All Unread Notifications', cnt_unread_notifications, ('Count of unread notifications.'), None), cnt_unread_old_notifications=NamedStat(\ 'Unread: Older than %s Days' % day_cnt_1, cnt_old_unread_notifications, ('Count of' ' notifications <b>older' ' than %d days</b>') % day_cnt_1, None), cnt_old_unread_notifications2=NamedStat(\ 'Unread: Older than %s Days' % day_cnt_2, cnt_old_unread_notifications2, ('Count of' ' notifications <b>older' ' than %d days</b>') % day_cnt_2, None), cnt_undated_notifications=NamedStat(\ 'Undated Notifications', cnt_undated_notifications, ('Count of undated notifications'), None), #cnt_harvested_zero=NamedStat(\ # 'Filesize 0 (Harvested)', # cnt_harvested_zero, # ('Count of harvested Datafiles displaying a' # ' size of 0 bytes')), ) return file_stats
def get_count_broken_notifications(): """ Query each object type and make sure notifications aren't broken Example map { 'DvObject': [1], 'Dataverse': [2], 'Dataset': [14, 11], 'DatasetVersion': [13, 12, 7], 'DataFile': [9] } """ broken_cnt = 0 user_ids = [] for model_name, type_id_list in get_dv_object_to_object_id_map().items(): # Get a list of object ids for this model type # that were not emailed--e.g. should show up # on the notifications pages # msgt('check: %s %s' % (model_name, type_id_list)) model_user_id_list = UserNotification.objects.select_related('user'\ ).filter(\ object_type__in=type_id_list, ).values_list('objectid', 'user__id') model_id_list = [x[0] for x in model_user_id_list] user_ids += [x[1] for x in model_user_id_list] msg('model_id_list len: %s' % len(model_id_list)) if len(model_id_list) == 0: continue # Used for later "bad notice" counts notice_counter = Counter(model_id_list) msg('notice_counter len: %s' % len(notice_counter)) unique_id_list = list(set(model_id_list)) msg('unique_id_list len: %s' % len(unique_id_list)) # Need to upgrade apps files and not use this method model_class = eval(model_name) if model_name in ['DvObject', 'DatasetVersion', 'FileMetadata']: existing_ids = model_class.objects.filter(id__in=unique_id_list\ ).values_list('id', flat=True\ ).distinct() else: existing_ids = model_class.objects.select_related('dvobject'\ ).filter(dvobject__id__in=unique_id_list\ ).values_list('dvobject__id', flat=True\ ).distinct() msg('existing_ids len: %s' % len(existing_ids)) if len(unique_id_list) == len(existing_ids): # Looks good! continue missing_ids = list(set(unique_id_list) - set(existing_ids)) for missing_id in missing_ids: broken_cnt += notice_counter.get(missing_id, 0) unique_user_ids = len(set(user_ids)) return (broken_cnt, unique_user_ids)
def make_json_files(self): # Set publication status # filters = {} if self.published_only: filters.update(query_helper.get_is_published_filter_param()) # Query for dataset ids # ds_id_query = Dataset.objects.filter(**filters\ ).annotate(ds_id=F('dvobject__id')\ ).values_list('ds_id', flat=True\ ).order_by('ds_id') # Iterate through dataset ids # #start_time = datetime.now() start_time = int(time.time()) # epoch seconds cnt = 0 no_versions_found_list = [45900] for ds_id in ds_id_query: cnt += 1 msgt('(%d) Checking dataset id %s' % (cnt, ds_id)) if ds_id < self.dataset_start_id: msg('skipping...(start at dataset id: %d)' % self.dataset_start_id) continue # Create file name # fname = 'ds_%s.json' % (str(ds_id).zfill(8)) full_fname = join(OUTPUT_DIR, fname) # Should we overwrite the existing file? # if isfile(full_fname) and not self.overwrite_existing_files: msg('skipping...file already exists') continue dataset_version = get_latest_dataset_version(ds_id) if dataset_version is None: msg("Could not find dataset_version!") no_versions_found_list.append(ds_id) continue dataset_as_json = DatasetSerializer(dataset_version).as_json() open(full_fname, 'w').write(json.dumps(dataset_as_json, indent=4)) msg('File written: %s' % full_fname) if cnt % 500 == 0: self.show_elapsed_time(start_time) #if cnt > 10: # self.show_elapsed_time(start_time) # break self.show_elapsed_time(start_time) print 'no_versions_found_list: %s' % no_versions_found_list
def get_basic_stats(): cnt_read_notifications = UserNotification.objects.filter(\ readnotification=True, ).count() cnt_unread_notifications = UserNotification.objects.filter(\ readnotification=False, ).count() cnt_undated_notifications = UserNotification.objects.filter(\ senddate__isnull=True ).count() day_cnt_1 = 365 day_cnt_1_date = datetime.now() - timedelta(days=day_cnt_1) day_cnt_2 = 180 day_cnt_2_date = datetime.now() - timedelta(days=day_cnt_2) cnt_old_unread_notifications = UserNotification.objects.filter(\ readnotification=False, senddate__lt=day_cnt_1_date ).count() cnt_old_unread_notifications2 = UserNotification.objects.filter(\ readnotification=False, senddate__lt=day_cnt_2_date ).count() broken_cnt, impacted_users = NotificationStats.get_count_broken_notifications( ) msg('broken_cnt: %s' % broken_cnt) msg('impacted_users: %s' % impacted_users) file_stats = dict(\ cnt_broken_notifications=NamedStat(\ 'Broken Notifications / Impacted Users', broken_cnt, ('The notification refers to an object that' ' no longer exists. These notifications should' ' be deleted from the database. (May be' ' responsible for some users who receive an' ' error when clicking on the notifications' ' tab.)'), 'view_broken_notifications', **dict(stat2=impacted_users)), cnt_read_notifications=NamedStat(\ 'Read Notifications', cnt_read_notifications, ('Count of read notifications'), None), cnt_unread_notifications=NamedStat(\ 'All Unread Notifications', cnt_unread_notifications, ('Count of unread notifications.'), None), cnt_unread_old_notifications=NamedStat(\ 'Unread: Older than %s Days' % day_cnt_1, cnt_old_unread_notifications, ('Count of' ' notifications <b>older' ' than %d days</b>') % day_cnt_1, None), cnt_old_unread_notifications2=NamedStat(\ 'Unread: Older than %s Days' % day_cnt_2, cnt_old_unread_notifications2, ('Count of' ' notifications <b>older' ' than %d days</b>') % day_cnt_2, None), cnt_undated_notifications=NamedStat(\ 'Undated Notifications', cnt_undated_notifications, ('Count of undated notifications'), None), #cnt_harvested_zero=NamedStat(\ # 'Filesize 0 (Harvested)', # cnt_harvested_zero, # ('Count of harvested Datafiles displaying a' # ' size of 0 bytes')), ) return file_stats
def get_data_rows(self, as_json=False, pretty_print=False): """ Return information as JSON { "data" : "total_row_count" : 117 "preview_row_count" : 50 "column_names" : ["Name", "Position", "Office"] "rows" : [ [ "Tiger Nixon", "System Architect", "Edinburgh" ], [ "Garrett Winters", "Accountant", "Tokyo" ] ] } """ if self.has_error(): return None # Read the table try: if self.is_excel: msgt('Excel!') df = pd.read_excel(self.filepath) #error_bad_lines=False) else: df = pd.read_table(self.filepath) except Exception as ex_obj: msg(ex_obj) msgt('Failed to open file via pandas!') temp_file_helper.make_sure_file_deleted(self.filepath) if self.is_excel: self.add_error('Failed to open Excel file via pandas. [%s]' % ex_obj) else: self.add_error( '<b>Probably not a tabular file!</b> Failed to open file via pandas. [%s]' % ex_obj) return None self.describe_as_html = df.describe().to_html() json_string = df.describe().to_json() self.describe_as_dict = json.loads(json_string, object_pairs_hook=OrderedDict) # Retrieve the columns self.column_names = df.columns.tolist() # Retrieve the rows self.data_rows = df[:self.num_preview_rows].values.tolist() #print 'rows', json.dumps(rows) # Format the response info_dict = OrderedDict() info_dict['total_row_count'] = len(df.index) info_dict['preview_row_count'] = len(self.data_rows) info_dict['column_names'] = self.column_names info_dict['rows'] = self.data_rows info_dict['describe_as_html'] = self.describe_as_html info_dict['describe_as_dict'] = self.describe_as_dict if as_json: if pretty_print: return json.dumps(info_dict, indent=4) return json.dumps(info_dict) return info_dict