def load_broken_notifications(self): """Load broken notifications by type""" broken_notice_info = None for model_name, type_id_list in get_dv_object_to_object_id_map().items(): # Get a list of object ids for this model type # that were not emailed--e.g. should show up # on the notifications pages # msgt('check: %s %s' % (model_name, type_id_list)) # If there's a selected_model_name, then only process that model # if self.selected_model_name is None: pass # check all models elif model_name != self.selected_model_name: # We have a selected_model_name and this isn't it! continue model_user_id_list = UserNotification.objects.select_related('user'\ ).filter(\ object_type__in=type_id_list, ).values_list('objectid', 'user__id') if len(model_user_id_list) == 0: continue # retrieve the object ids only model_id_list = [x[0] for x in model_user_id_list] unique_id_list = list(set(model_id_list)) # Next line is a hack - Need to upgrade Django apps # to not use this method model_class = eval(model_name) if model_name in ['DvObject', 'DatasetVersion', 'FileMetadata']: existing_ids = model_class.objects.filter(id__in=unique_id_list\ ).values_list('id', flat=True\ ).distinct() else: existing_ids = model_class.objects.select_related('dvobject'\ ).filter(dvobject__id__in=unique_id_list\ ).values_list('dvobject__id', flat=True\ ).distinct() if len(unique_id_list) == len(existing_ids): # Looks good! # No notifications where object no longer exists continue # Create a list of the missing ids # missing_ids = list(set(unique_id_list) - set(existing_ids)) # Record broken notification info # broken_notice_info = BrokenNotificationInfo(\ model_name, list(model_user_id_list), missing_ids) self.broken_info_list.append(broken_notice_info)
def download_file(url_to_file): """Download a Dataverse file and return the filename""" """ import re d = r.headers['content-disposition'] fname = re.findall("filename=(.+)", d) """ file_handle, filepath = tempfile.mkstemp() msgt('download file: %s' % url_to_file) r = requests.get(url_to_file, stream=True) if r.status_code != 200: msg('bad status: %s' % r.status_code) if isfile(filepath): make_sure_file_deleted(filepath) return None, None file_ext = None content_dict = r.headers['content-disposition'] #print 'content_dict', content_dict #fname = re.findall("filename=(.+)", content_dict) fname = format_file_name(content_dict) if fname: file_ext = fname.split('.')[-1].lower() print 'file_ext', file_ext with os.fdopen(file_handle, 'wb') as tmp: for chunk in r.iter_content(chunk_size=1024): if chunk: tmp.write(chunk) msg('File downloaded: %s' % filepath) return filepath, file_ext
def show_diffs(self): """Debug: print out differences""" section = None for diff_obj in self.diff_list: if section != diff_obj.section: section = diff_obj.section msgt(section) diff_obj.show(show_section=False)
def show_diffs(self): """Debug: print out differences""" section = None for diff_obj in self.diff_list: if section != diff_obj.section: section = diff_obj.section msgt(section) diff_obj.show(show_section=False)
def show(self, show_section=True): """print info""" if show_section: msgt('%s: [%s] %s' % (self.section, self.attr_name, self.note)) msg('attribute: %s' % self.attr_name) msg('\nnew: %s' % self.new_val) msg('\nold: %s' % self.old_val) dashes()
def show(self, show_section=True): """print info""" if show_section: msgt('%s: [%s] %s' % (self.section, self.attr_name, self.note)) msg('attribute: %s' % self.attr_name) msg('\nnew: %s' % self.new_val) msg('\nold: %s' % self.old_val) dashes()
def show_elapsed_time(self, start_time): """From http://stackoverflow.com/questions/1345827/how-do-i-find-the-time-difference-between-two-datetime-objects-in-python""" time_now = int(time.time()) # epoch seconds days = divmod(time_now - start_time, 86400) # days hours = divmod(days[1], 3600) # hours minutes = divmod(hours[1],60) # minutes seconds = minutes[1] # seconds msgt('Elapsed time: %d day(s), %d hour(s), %d minute(s), %d second(s)' % (days[0],hours[0],minutes[0],seconds))
def load_file_as_dict(fname): """Load a file as a python dict""" msgt('load file: %s' % fname) assert isfile(fname), '%s is not file' % fname fcontent = open(fname, 'r').read() fcontent = fcontent.replace('\r\n', '\\r\\n') dict_info = json.loads(fcontent, object_pairs_hook=OrderedDict) if dict_info.has_key('data'): return dict_info['data'] return dict_info
def load_file_as_dict(fname): """Load a file as a python dict""" msgt('load file: %s' % fname) assert isfile(fname), '%s is not file' % fname fcontent = open(fname, 'r').read() fcontent = fcontent.replace('\r\n', '\\r\\n') dict_info = json.loads(fcontent, object_pairs_hook=OrderedDict) if dict_info.has_key('data'): return dict_info['data'] return dict_info
def show_elapsed_time(self, start_time): """From http://stackoverflow.com/questions/1345827/how-do-i-find-the-time-difference-between-two-datetime-objects-in-python""" time_now = int(time.time()) # epoch seconds days = divmod(time_now - start_time, 86400) # days hours = divmod(days[1], 3600) # hours minutes = divmod(hours[1], 60) # minutes seconds = minutes[1] # seconds msgt( 'Elapsed time: %d day(s), %d hour(s), %d minute(s), %d second(s)' % (days[0], hours[0], minutes[0], seconds))
def write_files_to_mongo(self, **kwargs): """Write the saved dataset files to Mongo""" client = MongoClient() db = client.dataverse_database collection = db.datasets # look at kwargs # dataset_start_id = kwargs.get('dataset_start_id', 0) delete_all = kwargs.get('delete_all', False) # If appropriate, Delete existing records # if delete_all: msgt('Deleting current records') result = collection.delete_many({}) msg('result.deleted_count: %s' % result.deleted_count) return fnames = os.listdir(self.output_dir) fnames = [ x for x in fnames if x.endswith('.json') and x.startswith('ds_') ] fnames.sort() start_time = int(time.time()) # epoch seconds cnt = 0 for fname in fnames: cnt += 1 ds_id = int(fname.split('.')[0].split('_')[1]) msgt('(%d) process dataset %s (%s)' % (cnt, ds_id, fname)) if ds_id < dataset_start_id: msg('skipping it') continue content = open(join(self.output_dir, fname), 'r').read() content = update_json_text(content) content_doc = json.loads(content, object_pairs_hook=OrderedDict) content_doc['_id'] = ds_id content_doc['dtype'] = 'dataset' #doc_id = collection.insert_one(content_doc).inserted_id #doc_id = collection.save(content_doc) #.inserted_id doc_id = collection.save(content_doc) if cnt % 500 == 0: self.show_elapsed_time(start_time) self.show_elapsed_time(start_time)
def write_files_to_mongo(self, **kwargs): """Write the saved dataset files to Mongo""" client = MongoClient() db = client.dataverse_database collection = db.datasets # look at kwargs # dataset_start_id = kwargs.get('dataset_start_id', 0) delete_all = kwargs.get('delete_all', False) # If appropriate, Delete existing records # if delete_all: msgt('Deleting current records') result = collection.delete_many({}) msg('result.deleted_count: %s' % result.deleted_count) return fnames = os.listdir(self.output_dir) fnames = [x for x in fnames if x.endswith('.json') and x.startswith('ds_')] fnames.sort() start_time = int(time.time()) # epoch seconds cnt = 0 for fname in fnames: cnt += 1 ds_id = int(fname.split('.')[0].split('_')[1]) msgt('(%d) process dataset %s (%s)' % (cnt, ds_id, fname)) if ds_id < dataset_start_id: msg('skipping it') continue content = open(join(self.output_dir, fname), 'r').read() content = update_json_text(content) content_doc = json.loads(content, object_pairs_hook=OrderedDict) content_doc['_id'] = ds_id content_doc['dtype'] = 'dataset' #doc_id = collection.insert_one(content_doc).inserted_id #doc_id = collection.save(content_doc) #.inserted_id doc_id = collection.save(content_doc) if cnt % 500 == 0: self.show_elapsed_time(start_time) self.show_elapsed_time(start_time)
def test_search_mongo(self, term='law'): """Test searches""" client = MongoClient() db = client.dataverse_database collection = db.datasets # Compass: # # {"title": {$regex: "(^Law| Law | Law$)"}} """ {"title":{"$regex":"(^Law| Law | Law$)","$options":"i"},"metadata_blocks.citation.dsDescription.dsDescriptionValue": {"$regex":"(^Law| Law | Law$)","$options":"i"}} """ field_names = [ 'metadata_blocks.citation.dsDescription.dsDescriptionValue', #'title', #'metadata_blocks.citation.subject', #'metadata_blocks.citation.keyword.keywordValue', ] qlist = [] for field_name in field_names: qlist.append({ field_name: { '$regex': '(^{0}|\s{0}\s|\s{0}$)'.format(term), '$options': 'i' } }) docs = collection.find({"$or": qlist}) # ----------------------------- #field_name = 'title' #field_name = 'metadata_blocks.citation.dsDescription.dsDescriptionValue' #docs = collection.find({field_name:{'$regex':'(^Law|\sLaw\s|\sLaw$)', '$options':'i'}}) #docs = collection.find({'title':{'$regex':'(^Law|\sLaw\s|\sLaw$)', '$options':'i'}}) from dict_map_util import DictMapUtil cnt = 0 for doc in docs: cnt += 1 msgt('(%d) %s' % (cnt, doc['title'])) dmap_str = 'dmap.' + field_names[0] print 'dmap_str', dmap_str m = DictMapUtil(doc) import ipdb ipdb.set_trace() #print eval(dmap_str) break """
def test_search_mongo(self, term='law'): """Test searches""" client = MongoClient() db = client.dataverse_database collection = db.datasets # Compass: # # {"title": {$regex: "(^Law| Law | Law$)"}} """ {"title":{"$regex":"(^Law| Law | Law$)","$options":"i"},"metadata_blocks.citation.dsDescription.dsDescriptionValue": {"$regex":"(^Law| Law | Law$)","$options":"i"}} """ field_names = [ 'metadata_blocks.citation.dsDescription.dsDescriptionValue', #'title', #'metadata_blocks.citation.subject', #'metadata_blocks.citation.keyword.keywordValue', ] qlist = [] for field_name in field_names: qlist.append({ field_name: {'$regex':'(^{0}|\s{0}\s|\s{0}$)'.format(term), '$options':'i'}} ) docs = collection.find({"$or": qlist}) # ----------------------------- #field_name = 'title' #field_name = 'metadata_blocks.citation.dsDescription.dsDescriptionValue' #docs = collection.find({field_name:{'$regex':'(^Law|\sLaw\s|\sLaw$)', '$options':'i'}}) #docs = collection.find({'title':{'$regex':'(^Law|\sLaw\s|\sLaw$)', '$options':'i'}}) from dict_map_util import DictMapUtil cnt = 0 for doc in docs: cnt += 1 msgt('(%d) %s' % (cnt, doc['title'])) dmap_str = 'dmap.' + field_names[0] print 'dmap_str', dmap_str m = DictMapUtil(doc) #print eval(dmap_str) break """
def run_comparison(self): """Compare the two JSON datasets""" msgt(self.run_comparison.__doc__) # Run a quick check to see if the dicts are the same. # if cmp(self.old_ds, self.new_ds) == 0: msg('No differences!') return new_files_list = self.new_ds.pop('files', []) old_files_list = self.old_ds.pop('files', []) #print 'new_files_list', new_files_list self.compare_dicts(\ '', self.new_ds, self.old_ds) self.compare_file_lists(\ new_files_list, old_files_list)
def download_file(url_to_file): """Download a Dataverse file and return the filename""" file_handle, filepath = tempfile.mkstemp() msgt('download file: %s' % url_to_file) r = requests.get(url_to_file, stream=True) if r.status_code != 200: msg('bad status: %s' % r.status_code) if isfile(filepath): make_sure_file_deleted(filepath) return None with os.fdopen(file_handle, 'wb') as tmp: for chunk in r.iter_content(chunk_size=1024): if chunk: tmp.write(chunk) msg('File downloaded: %s' % filepath) return filepath
def run_comparison(self): """Compare the two JSON datasets""" msgt(self.run_comparison.__doc__) # Run a quick check to see if the dicts are the same. # if cmp(self.old_ds, self.new_ds) == 0: msg('No differences!') return new_files_list = self.new_ds.pop('files', []) old_files_list = self.old_ds.pop('files', []) #print 'new_files_list', new_files_list self.compare_dicts(\ '', self.new_ds, self.old_ds) self.compare_file_lists(\ new_files_list, old_files_list)
def download_file(url_to_file): """Download a Dataverse file and return the filename""" """ import re d = r.headers['content-disposition'] fname = re.findall("filename=(.+)", d) """ file_handle, filepath = tempfile.mkstemp() msgt('download file: %s' % url_to_file) r = requests.get(url_to_file, stream=True) if r.status_code != 200: msg('bad status: %s' % r.status_code) if isfile(filepath): make_sure_file_deleted(filepath) return None, None file_ext = None content_dict = r.headers['content-disposition'] #print 'content_dict', content_dict #fname = re.findall("filename=(.+)", content_dict) fname = format_file_name(content_dict) if fname: file_ext = fname.split('.')[-1].lower() print 'file_ext', file_ext with os.fdopen(file_handle, 'wb') as tmp: for chunk in r.iter_content(chunk_size=1024): if chunk: tmp.write(chunk) msg('File downloaded: %s' % filepath) return filepath, file_ext
def make_json_files(self): # Set publication status # filters = {} if self.published_only: filters.update(query_helper.get_is_published_filter_param()) # Query for dataset ids # ds_id_query = Dataset.objects.filter(**filters\ ).annotate(ds_id=F('dvobject__id')\ ).values_list('ds_id', flat=True\ ).order_by('ds_id') # Iterate through dataset ids # #start_time = datetime.now() start_time = int(time.time()) # epoch seconds cnt = 0 no_versions_found_list = [45900] for ds_id in ds_id_query: cnt += 1 msgt('(%d) Checking dataset id %s' % (cnt, ds_id)) if ds_id < self.dataset_start_id: msg('skipping...(start at dataset id: %d)' % self.dataset_start_id) continue # Create file name # fname = 'ds_%s.json' % (str(ds_id).zfill(8)) full_fname = join(OUTPUT_DIR, fname) # Should we overwrite the existing file? # if isfile(full_fname) and not self.overwrite_existing_files: msg('skipping...file already exists') continue dataset_version = get_latest_dataset_version(ds_id) if dataset_version is None: msg("Could not find dataset_version!") no_versions_found_list.append(ds_id) continue dataset_as_json = DatasetSerializer(dataset_version).as_json() open(full_fname, 'w').write(json.dumps(dataset_as_json, indent=4)) msg('File written: %s' % full_fname) if cnt % 500 == 0: self.show_elapsed_time(start_time) #if cnt > 10: # self.show_elapsed_time(start_time) # break self.show_elapsed_time(start_time) print 'no_versions_found_list: %s' % no_versions_found_list
def load_broken_notifications(self): """Load broken notifications by type""" broken_notice_info = None for model_name, type_id_list in get_dv_object_to_object_id_map().items( ): # Get a list of object ids for this model type # that were not emailed--e.g. should show up # on the notifications pages # msgt('check: %s %s' % (model_name, type_id_list)) # If there's a selected_model_name, then only process that model # if self.selected_model_name is None: pass # check all models elif model_name != self.selected_model_name: # We have a selected_model_name and this isn't it! continue model_user_id_list = UserNotification.objects.select_related('user'\ ).filter(\ object_type__in=type_id_list, ).values_list('objectid', 'user__id') if len(model_user_id_list) == 0: continue # retrieve the object ids only model_id_list = [x[0] for x in model_user_id_list] unique_id_list = list(set(model_id_list)) # Next line is a hack - Need to upgrade Django apps # to not use this method model_class = eval(model_name) if model_name in ['DvObject', 'DatasetVersion', 'FileMetadata']: existing_ids = model_class.objects.filter(id__in=unique_id_list\ ).values_list('id', flat=True\ ).distinct() else: existing_ids = model_class.objects.select_related('dvobject'\ ).filter(dvobject__id__in=unique_id_list\ ).values_list('dvobject__id', flat=True\ ).distinct() if len(unique_id_list) == len(existing_ids): # Looks good! # No notifications where object no longer exists continue # Create a list of the missing ids # missing_ids = list(set(unique_id_list) - set(existing_ids)) # Record broken notification info # broken_notice_info = BrokenNotificationInfo(\ model_name, list(model_user_id_list), missing_ids) self.broken_info_list.append(broken_notice_info)
def get_count_broken_notifications(): """ Query each object type and make sure notifications aren't broken Example map { 'DvObject': [1], 'Dataverse': [2], 'Dataset': [14, 11], 'DatasetVersion': [13, 12, 7], 'DataFile': [9] } """ broken_cnt = 0 user_ids = [] for model_name, type_id_list in get_dv_object_to_object_id_map().items( ): # Get a list of object ids for this model type # that were not emailed--e.g. should show up # on the notifications pages # msgt('check: %s %s' % (model_name, type_id_list)) model_user_id_list = UserNotification.objects.select_related('user'\ ).filter(\ object_type__in=type_id_list, ).values_list('objectid', 'user__id') model_id_list = [x[0] for x in model_user_id_list] user_ids += [x[1] for x in model_user_id_list] msg('model_id_list len: %s' % len(model_id_list)) if len(model_id_list) == 0: continue # Used for later "bad notice" counts notice_counter = Counter(model_id_list) msg('notice_counter len: %s' % len(notice_counter)) unique_id_list = list(set(model_id_list)) msg('unique_id_list len: %s' % len(unique_id_list)) # Need to upgrade apps files and not use this method model_class = eval(model_name) if model_name in ['DvObject', 'DatasetVersion', 'FileMetadata']: existing_ids = model_class.objects.filter(id__in=unique_id_list\ ).values_list('id', flat=True\ ).distinct() else: existing_ids = model_class.objects.select_related('dvobject'\ ).filter(dvobject__id__in=unique_id_list\ ).values_list('dvobject__id', flat=True\ ).distinct() msg('existing_ids len: %s' % len(existing_ids)) if len(unique_id_list) == len(existing_ids): # Looks good! continue missing_ids = list(set(unique_id_list) - set(existing_ids)) for missing_id in missing_ids: broken_cnt += notice_counter.get(missing_id, 0) unique_user_ids = len(set(user_ids)) return (broken_cnt, unique_user_ids)
def make_json_files(self): # Set publication status # filters = {} if self.published_only: filters.update(query_helper.get_is_published_filter_param()) # Query for dataset ids # ds_id_query = Dataset.objects.filter(**filters\ ).annotate(ds_id=F('dvobject__id')\ ).values_list('ds_id', flat=True\ ).order_by('ds_id') # Iterate through dataset ids # #start_time = datetime.now() start_time = int(time.time()) # epoch seconds cnt = 0 no_versions_found_list = [45900] for ds_id in ds_id_query: cnt += 1 msgt('(%d) Checking dataset id %s' % (cnt, ds_id)) if ds_id < self.dataset_start_id: msg('skipping...(start at dataset id: %d)' % self.dataset_start_id) continue # Create file name # fname = 'ds_%s.json' % (str(ds_id).zfill(8)) full_fname = join(OUTPUT_DIR, fname) # Should we overwrite the existing file? # if isfile(full_fname) and not self.overwrite_existing_files: msg('skipping...file already exists') continue dataset_version = get_latest_dataset_version(ds_id) if dataset_version is None: msg("Could not find dataset_version!") no_versions_found_list.append(ds_id) continue dataset_as_json = DatasetSerializer(dataset_version).as_json() open(full_fname, 'w').write(json.dumps(dataset_as_json, indent=4)) msg('File written: %s' % full_fname) if cnt % 500 == 0: self.show_elapsed_time(start_time) #if cnt > 10: # self.show_elapsed_time(start_time) # break self.show_elapsed_time(start_time) print 'no_versions_found_list: %s' % no_versions_found_list
def get_data_rows(self, as_json=False, pretty_print=False): """ Return information as JSON { "data" : "total_row_count" : 117 "preview_row_count" : 50 "column_names" : ["Name", "Position", "Office"] "rows" : [ [ "Tiger Nixon", "System Architect", "Edinburgh" ], [ "Garrett Winters", "Accountant", "Tokyo" ] ] } """ if self.has_error(): return None # Read the table try: if self.is_excel: msgt('Excel!') df = pd.read_excel(self.filepath) #error_bad_lines=False) else: df = pd.read_table(self.filepath) except Exception as ex_obj: msg(ex_obj) msgt('Failed to open file via pandas!') temp_file_helper.make_sure_file_deleted(self.filepath) if self.is_excel: self.add_error('Failed to open Excel file via pandas. [%s]' % ex_obj) else: self.add_error( '<b>Probably not a tabular file!</b> Failed to open file via pandas. [%s]' % ex_obj) return None self.describe_as_html = df.describe().to_html() json_string = df.describe().to_json() self.describe_as_dict = json.loads(json_string, object_pairs_hook=OrderedDict) # Retrieve the columns self.column_names = df.columns.tolist() # Retrieve the rows self.data_rows = df[:self.num_preview_rows].values.tolist() #print 'rows', json.dumps(rows) # Format the response info_dict = OrderedDict() info_dict['total_row_count'] = len(df.index) info_dict['preview_row_count'] = len(self.data_rows) info_dict['column_names'] = self.column_names info_dict['rows'] = self.data_rows info_dict['describe_as_html'] = self.describe_as_html info_dict['describe_as_dict'] = self.describe_as_dict if as_json: if pretty_print: return json.dumps(info_dict, indent=4) return json.dumps(info_dict) return info_dict
def get_count_broken_notifications(): """ Query each object type and make sure notifications aren't broken Example map { 'DvObject': [1], 'Dataverse': [2], 'Dataset': [14, 11], 'DatasetVersion': [13, 12, 7], 'DataFile': [9] } """ broken_cnt = 0 user_ids = [] for model_name, type_id_list in get_dv_object_to_object_id_map().items(): # Get a list of object ids for this model type # that were not emailed--e.g. should show up # on the notifications pages # msgt('check: %s %s' % (model_name, type_id_list)) model_user_id_list = UserNotification.objects.select_related('user'\ ).filter(\ object_type__in=type_id_list, ).values_list('objectid', 'user__id') model_id_list = [x[0] for x in model_user_id_list] user_ids += [x[1] for x in model_user_id_list] msg('model_id_list len: %s' % len(model_id_list)) if len(model_id_list) == 0: continue # Used for later "bad notice" counts notice_counter = Counter(model_id_list) msg('notice_counter len: %s' % len(notice_counter)) unique_id_list = list(set(model_id_list)) msg('unique_id_list len: %s' % len(unique_id_list)) # Need to upgrade apps files and not use this method model_class = eval(model_name) if model_name in ['DvObject', 'DatasetVersion', 'FileMetadata']: existing_ids = model_class.objects.filter(id__in=unique_id_list\ ).values_list('id', flat=True\ ).distinct() else: existing_ids = model_class.objects.select_related('dvobject'\ ).filter(dvobject__id__in=unique_id_list\ ).values_list('dvobject__id', flat=True\ ).distinct() msg('existing_ids len: %s' % len(existing_ids)) if len(unique_id_list) == len(existing_ids): # Looks good! continue missing_ids = list(set(unique_id_list) - set(existing_ids)) for missing_id in missing_ids: broken_cnt += notice_counter.get(missing_id, 0) unique_user_ids = len(set(user_ids)) return (broken_cnt, unique_user_ids)