Example #1
0
 def store_records(self, act_table, recs):
     """
     stores records retrieved for a given table
     """
     i = 0
     for record in recs:
         i += 1
         allow_write = self.check_allow_write(act_table, record)
         record = self.prep_update_keep_old(act_table, record)
         if (allow_write is False and self.update_keep_old is False):
             print('\n Not allowed to overwite record.' + str(i))
         else:
             # print('\n Adding record:' + str(record))
             newr = False
             if (act_table == 'link_annotations'):
                 newr = LinkAnnotation(**record)
             elif (act_table == 'link_entities'):
                 newr = LinkEntity(**record)
             elif (act_table == 'oc_assertions'):
                 newr = Assertion(**record)
             elif (act_table == 'oc_manifest'):
                 newr = Manifest(**record)
             elif (act_table == 'oc_subjects'):
                 newr = Subject(**record)
             elif (act_table == 'oc_mediafiles'):
                 newr = Mediafile(**record)
             elif (act_table == 'oc_documents'):
                 newr = OCdocument(**record)
             elif (act_table == 'oc_persons'):
                 newr = Person(**record)
             elif (act_table == 'oc_projects'):
                 newr = Project(**record)
             elif (act_table == 'oc_strings'):
                 newr = OCstring(**record)
             elif (act_table == 'oc_types'):
                 newr = OCtype(**record)
             elif (act_table == 'oc_geospace'):
                 newr = Geospace(**record)
             elif (act_table == 'oc_events'):
                 newr = Event(**record)
             elif (act_table == 'oc_predicates'):
                 newr = Predicate(**record)
             elif (act_table == 'oc_identifiers'):
                 newr = StableIdentifer(**record)
             elif (act_table == 'oc_obsmetadata'):
                 newr = ObsMetadata(**record)
             if (newr is not False):
                 try:
                     newr.save(force_insert=self.force_insert,
                               force_update=self.update_keep_old)
                 except Exception as error:
                     print('Something slipped past in ' + act_table +
                           '...' + str(error))
Example #2
0
 def add_item_stable_id(self, post_data):
     """ adds a stable identifier to an item """
     ok = False
     note = ''
     orcid_ok = self.check_orcid_ok(post_data)
     stable_id = self.request_param_val(post_data,
                                        'stable_id')
     stable_type = self.request_param_val(post_data,
                                          'stable_type')
     stable_id = stable_id.strip()
     # now update the stable_type based on what's in the stable_id
     stable_type = StableIdentifer().type_uri_check(stable_type,
                                                    stable_id)
     if stable_type == 'orcid' and orcid_ok is not True:
         # problem adding an ORCID to this type of item
         stable_type = False
         note = 'Cannot add an ORCID to this item.'
     if stable_id is not False \
        and stable_type is not False:
         ok = True
         try:
             new_stable = StableIdentifer()
             new_stable.stable_id = stable_id
             new_stable.stable_type = stable_type
             new_stable.uuid = self.manifest.uuid
             new_stable.project_uuid = self.manifest.project_uuid
             new_stable.item_type = self.manifest.item_type
             new_stable.save()
         except:
             ok = False
             note = 'Identifier already in use'
     else:
         note = 'Problems with the ID request'
     if ok:
         # now clear the cache a change was made
         self.clear_caches()
     self.response = {'action': 'add-item-stable-id',
                      'ok': ok,
                      'change': {'note': note}}
     return self.response
Example #3
0
 def get_table_dois(self):
     """ gets a list of tables from the tab-manifest.csv directory """
     tab_obj = self.load_csv_file(self.table_dir, self.table_manifest_csv)
     if tab_obj is not False:
         fields = self.get_table_fields(tab_obj)
         # first field is the tableID
         i = 0
         for row in tab_obj:
             if i > 0:
                 meta = {}
                 meta['table_id'] = row[0]  # first cell has the table ID field
                 meta['label'] = row[3]
                 meta['created'] = row[4]
                 row_len = len(row)
                 ids = {'doi': None,
                        'ark': None}
                 f_i = 12
                 last_f_i = row_len - 1
                 while f_i <= last_f_i:
                     json_part = row[f_i].replace('\\', '')
                     if '"doi":' in json_part and \
                        '"doi":"http://' not in json_part:
                         doi = json_part.replace('"doi":', '')
                         doi = doi.replace('"', '')
                         ids['doi'] = doi
                         print('DOI: ' + doi)
                     if '"ark":' in json_part and \
                        '"ark":"http://' not in json_part:
                         ark = json_part.replace('"ark":', '')
                         ark = ark.replace('"', '')
                         ids['ark'] = ark
                         print('ark: ' + ark)
                     f_i += 1
                 for stable_type, ident in ids.items():
                     if ident is not None:
                         id_obj = StableIdentifer()
                         id_obj.stable_id = ident
                         id_obj.stable_type = stable_type
                         id_obj.uuid = meta['table_id']
                         id_obj.project_uuid = '0'
                         id_obj.item_type = 'tables'
                         id_obj.save()
             i += 1
     return self.tab_metadata
Example #4
0
 def load_csv(self, filename, after=0, add_path=False):
     """ loads CSV dump from Merritt """
     if add_path:
         filename_path = os.path.join(settings.STATIC_ROOT,
                                      self.DEFAULT_DIRECTORY, filename)
     else:
         filename_path = filename
     data = csv.reader(open(filename_path))
     i = 0
     for row in data:
         manifest = False
         if 'ark:/' in row[0]:
             i += 0
             if i >= after:
                 uuid = URImanagement.get_uuid_from_oc_uri(row[1])
                 if uuid is not False:
                     try:
                         manifest = Manifest.objects.get(
                             uuid=uuid, archived__isnull=True)
                     except Manifest.DoesNotExist:
                         manifest = False
                 if manifest is not False:
                     ok_new = True
                     try:
                         sid = StableIdentifer()
                         sid.stable_id = row[0].replace('ark:/', '')
                         sid.stable_type = 'ark'
                         sid.uuid = manifest.uuid
                         sid.project_uuid = manifest.project_uuid
                         sid.item_type = manifest.item_type
                         sid.save()
                     except:
                         ok_new = False
                     # note when the item was last archived
                     try:
                         manifest.archived = self.validate_date(row[3])
                         manifest.archived_save()
                     except:
                         manifest.archived = time.strftime(
                             '%Y-%m-%d %H:%M:%S')
                         manifest.archived_save()
                     if ok_new:
                         self.id_recorded += 1
                     print('Saved ids: ' + str(self.id_recorded))
Example #5
0
 def store_records(self, act_table, recs):
     """
     stores records retrieved for a given table
     """
     for rkey, record in recs.items():
         if (act_table == 'link_annotations'):
             newr = LinkAnnotation(**record)
             newr.save()
         elif (act_table == 'link_entities'):
             newr = LinkEntity(**record)
             newr.save()
         elif (act_table == 'link_hierarchies'):
             newr = LinkHierarchy(**record)
             newr.save()
         elif (act_table == 'oc_chronology'):
             newr = Chronology(**record)
             newr.save()
         elif (act_table == 'oc_geodata'):
             newr = Geodata(**record)
             newr.save()
         elif (act_table == 'oc_mediafiles'):
             newr = Mediafile(**record)
             newr.save()
         elif (act_table == 'oc_documents'):
             newr = OCdocument(**record)
             newr.save()
         elif (act_table == 'oc_persons'):
             newr = Person(**record)
             newr.save()
         elif (act_table == 'oc_projects'):
             newr = Project(**record)
             newr.save()
         elif (act_table == 'oc_strings'):
             newr = OCstring(**record)
             newr.save()
         elif (act_table == 'oc_types'):
             newr = OCtype(**record)
             newr.save()
         elif (act_table == 'oc_events'):
             newr = Event(**record)
             newr.save()
         elif (act_table == 'oc_predicates'):
             newr = Predicate(**record)
             newr.save()
         elif (act_table == 'oc_identifiers'):
             newr = StableIdentifer(**record)
             newr.save()
         elif (act_table == 'oc_obsmetadata'):
             newr = ObsMetadata(**record)
             newr.save()
Example #6
0
 def load_csv(self, filename, after=0, add_path=False):
     """ loads CSV dump from Merritt """
     if add_path:
         filename_path = os.path.join(settings.STATIC_ROOT,
                                      self.DEFAULT_DIRECTORY,
                                      filename)
     else:
         filename_path = filename
     data = csv.reader(open(filename_path))
     i = 0
     for row in data:
         manifest = False
         if 'ark:/' in row[0]:
             i += 0
             if i >= after:
                 uuid = URImanagement.get_uuid_from_oc_uri(row[1])
                 if uuid is not False:
                     try:
                         manifest = Manifest.objects.get(uuid=uuid,
                                                         archived__isnull=True)
                     except Manifest.DoesNotExist:
                         manifest = False
                 if manifest is not False:
                     ok_new = True
                     try:
                         sid = StableIdentifer()
                         sid.stable_id = row[0].replace('ark:/', '')
                         sid.stable_type = 'ark'
                         sid.uuid = manifest.uuid
                         sid.project_uuid = manifest.project_uuid
                         sid.item_type = manifest.item_type
                         sid.save()
                     except:
                         ok_new = False
                     # note when the item was last archived
                     try:
                         manifest.archived = self.validate_date(row[3])
                         manifest.archived_save()
                     except:
                         manifest.archived = time.strftime('%Y-%m-%d %H:%M:%S')
                         manifest.archived_save()
                     if ok_new:
                         self.id_recorded += 1
                     print('Saved ids: ' + str(self.id_recorded))
Example #7
0
 def add_ids(self, ids):
     """ Adds ids to the database
     """
     if isinstance(ids, list):
         for id_rec in ids:
             id_and_type = self.parse_stable_id(id_rec['stable_id'])
             manifest = False
             uuid = URImanagement.get_uuid_from_oc_uri(id_rec['id'])
             if uuid is not False and id_and_type is not False:
                 try:
                     manifest = Manifest.objects.get(uuid=uuid)
                 except Manifest.DoesNotExist:
                     manifest = False
             if manifest is not False:
                 # we found the archived item in the manifest
                 # save the stable identifier in the database
                 ok_new = True
                 try:
                     sid = StableIdentifer()
                     sid.stable_id = id_and_type['id']
                     sid.stable_type = id_and_type['type']
                     sid.uuid = manifest.uuid
                     sid.project_uuid = manifest.project_uuid
                     sid.item_type = manifest.item_type
                     sid.save()
                 except:
                     ok_new = False
                 # note when the item was last archived
                 try:
                     manifest.archived = self.validate_date(
                         id_rec['archived'])
                     manifest.archived_save()
                 except:
                     manifest.archived = time.strftime('%Y-%m-%d %H:%M:%S')
                     manifest.archived_save()
                 if ok_new:
                     self.id_recorded += 1
     return self.id_recorded
Example #8
0
 def save_oc_item_stable_id(self, oc_item, stable_id, stable_type='ark'):
     """ saves stable_id for an oc_item object """
     stable_id = stable_id.strip()
     if 'doi:' in stable_id:
         stable_type = 'doi'
         stable_id = stable_id.replace('doi:', '')
     elif 'ark:/' in stable_id:
         stable_type = 'ark'
         stable_id = stable_id.replace('ark:/', '')
     print('Try to save ' + stable_type + ': ' + stable_id)
     try:
         ok = True
         new_stable = StableIdentifer()
         new_stable.stable_id = stable_id
         new_stable.stable_type = stable_type
         new_stable.uuid = oc_item.manifest.uuid
         new_stable.project_uuid = oc_item.manifest.project_uuid
         new_stable.item_type = oc_item.manifest.item_type
         new_stable.save()
     except:
         ok = False
         note = 'Identifier of type [' + stable_type + '] already exists.'
     return ok
Example #9
0
 def save_oc_item_stable_id(self, oc_item, stable_id, stable_type='ark'):
     """ saves stable_id for an oc_item object """
     stable_id = stable_id.strip()
     if 'doi:' in stable_id:
         stable_type = 'doi'
         stable_id = stable_id.replace('doi:', '')
     elif 'ark:/' in stable_id:
         stable_type = 'ark'
         stable_id = stable_id.replace('ark:/', '')
     print('Try to save ' + stable_type + ': ' + stable_id)
     try:
         ok = True
         new_stable = StableIdentifer()
         new_stable.stable_id = stable_id
         new_stable.stable_type = stable_type
         new_stable.uuid = oc_item.manifest.uuid
         new_stable.project_uuid = oc_item.manifest.project_uuid
         new_stable.item_type = oc_item.manifest.item_type
         new_stable.save()
     except:
         ok = False
         note = 'Identifier of type [' + stable_type + '] already exists.'
     return ok
Example #10
0
 def add_ids(self, ids):
     """ Adds ids to the database
     """
     if isinstance(ids, list):
         for id_rec in ids:
             id_and_type = self.parse_stable_id(id_rec['stable_id'])
             manifest = False
             uuid = URImanagement.get_uuid_from_oc_uri(id_rec['id'])
             if uuid is not False and id_and_type is not False:
                 try:
                     manifest = Manifest.objects.get(uuid=uuid)
                 except Manifest.DoesNotExist:
                     manifest = False
             if manifest is not False:
                 # we found the archived item in the manifest
                 # save the stable identifier in the database
                 ok_new = True
                 try:
                     sid = StableIdentifer()
                     sid.stable_id = id_and_type['id']
                     sid.stable_type = id_and_type['type']
                     sid.uuid = manifest.uuid
                     sid.project_uuid = manifest.project_uuid
                     sid.item_type = manifest.item_type
                     sid.save()
                 except:
                     ok_new = False
                 # note when the item was last archived
                 try:
                     manifest.archived = self.validate_date(id_rec['archived'])
                     manifest.archived_save()
                 except:
                     manifest.archived = time.strftime('%Y-%m-%d %H:%M:%S')
                     manifest.archived_save()
                 if ok_new:
                     self.id_recorded += 1
     return self.id_recorded