def store_records(self, act_table, recs):
"""
stores records retrieved for a given table
"""
i = 0
for record in recs:
i += 1
allow_write = self.check_allow_write(act_table, record)
record = self.prep_update_keep_old(act_table, record)
if (allow_write is False and self.update_keep_old is False):
print('\n Not allowed to overwite record.' + str(i))
else:
# print('\n Adding record:' + str(record))
newr = False
if (act_table == 'link_annotations'):
newr = LinkAnnotation(**record)
elif (act_table == 'link_entities'):
newr = LinkEntity(**record)
elif (act_table == 'oc_assertions'):
newr = Assertion(**record)
elif (act_table == 'oc_manifest'):
newr = Manifest(**record)
elif (act_table == 'oc_subjects'):
newr = Subject(**record)
elif (act_table == 'oc_mediafiles'):
newr = Mediafile(**record)
elif (act_table == 'oc_documents'):
newr = OCdocument(**record)
elif (act_table == 'oc_persons'):
newr = Person(**record)
elif (act_table == 'oc_projects'):
newr = Project(**record)
elif (act_table == 'oc_strings'):
newr = OCstring(**record)
elif (act_table == 'oc_types'):
newr = OCtype(**record)
elif (act_table == 'oc_geospace'):
newr = Geospace(**record)
elif (act_table == 'oc_events'):
newr = Event(**record)
elif (act_table == 'oc_predicates'):
newr = Predicate(**record)
elif (act_table == 'oc_identifiers'):
newr = StableIdentifer(**record)
elif (act_table == 'oc_obsmetadata'):
newr = ObsMetadata(**record)
if (newr is not False):
try:
newr.save(force_insert=self.force_insert,
force_update=self.update_keep_old)
except Exception as error:
print('Something slipped past in ' + act_table +
'...' + str(error))
def add_item_stable_id(self, post_data):
""" adds a stable identifier to an item """
ok = False
note = ''
orcid_ok = self.check_orcid_ok(post_data)
stable_id = self.request_param_val(post_data,
'stable_id')
stable_type = self.request_param_val(post_data,
'stable_type')
stable_id = stable_id.strip()
# now update the stable_type based on what's in the stable_id
stable_type = StableIdentifer().type_uri_check(stable_type,
stable_id)
if stable_type == 'orcid' and orcid_ok is not True:
# problem adding an ORCID to this type of item
stable_type = False
note = 'Cannot add an ORCID to this item.'
if stable_id is not False \
and stable_type is not False:
ok = True
try:
new_stable = StableIdentifer()
new_stable.stable_id = stable_id
new_stable.stable_type = stable_type
new_stable.uuid = self.manifest.uuid
new_stable.project_uuid = self.manifest.project_uuid
new_stable.item_type = self.manifest.item_type
new_stable.save()
except:
ok = False
note = 'Identifier already in use'
else:
note = 'Problems with the ID request'
if ok:
# now clear the cache a change was made
self.clear_caches()
self.response = {'action': 'add-item-stable-id',
'ok': ok,
'change': {'note': note}}
return self.response
def get_table_dois(self):
""" gets a list of tables from the tab-manifest.csv directory """
tab_obj = self.load_csv_file(self.table_dir, self.table_manifest_csv)
if tab_obj is not False:
fields = self.get_table_fields(tab_obj)
# first field is the tableID
i = 0
for row in tab_obj:
if i > 0:
meta = {}
meta['table_id'] = row[0] # first cell has the table ID field
meta['label'] = row[3]
meta['created'] = row[4]
row_len = len(row)
ids = {'doi': None,
'ark': None}
f_i = 12
last_f_i = row_len - 1
while f_i <= last_f_i:
json_part = row[f_i].replace('\\', '')
if '"doi":' in json_part and \
'"doi":"http://' not in json_part:
doi = json_part.replace('"doi":', '')
doi = doi.replace('"', '')
ids['doi'] = doi
print('DOI: ' + doi)
if '"ark":' in json_part and \
'"ark":"http://' not in json_part:
ark = json_part.replace('"ark":', '')
ark = ark.replace('"', '')
ids['ark'] = ark
print('ark: ' + ark)
f_i += 1
for stable_type, ident in ids.items():
if ident is not None:
id_obj = StableIdentifer()
id_obj.stable_id = ident
id_obj.stable_type = stable_type
id_obj.uuid = meta['table_id']
id_obj.project_uuid = '0'
id_obj.item_type = 'tables'
id_obj.save()
i += 1
return self.tab_metadata
def load_csv(self, filename, after=0, add_path=False):
""" loads CSV dump from Merritt """
if add_path:
filename_path = os.path.join(settings.STATIC_ROOT,
self.DEFAULT_DIRECTORY, filename)
else:
filename_path = filename
data = csv.reader(open(filename_path))
i = 0
for row in data:
manifest = False
if 'ark:/' in row[0]:
i += 0
if i >= after:
uuid = URImanagement.get_uuid_from_oc_uri(row[1])
if uuid is not False:
try:
manifest = Manifest.objects.get(
uuid=uuid, archived__isnull=True)
except Manifest.DoesNotExist:
manifest = False
if manifest is not False:
ok_new = True
try:
sid = StableIdentifer()
sid.stable_id = row[0].replace('ark:/', '')
sid.stable_type = 'ark'
sid.uuid = manifest.uuid
sid.project_uuid = manifest.project_uuid
sid.item_type = manifest.item_type
sid.save()
except:
ok_new = False
# note when the item was last archived
try:
manifest.archived = self.validate_date(row[3])
manifest.archived_save()
except:
manifest.archived = time.strftime(
'%Y-%m-%d %H:%M:%S')
manifest.archived_save()
if ok_new:
self.id_recorded += 1
print('Saved ids: ' + str(self.id_recorded))
def store_records(self, act_table, recs):
"""
stores records retrieved for a given table
"""
for rkey, record in recs.items():
if (act_table == 'link_annotations'):
newr = LinkAnnotation(**record)
newr.save()
elif (act_table == 'link_entities'):
newr = LinkEntity(**record)
newr.save()
elif (act_table == 'link_hierarchies'):
newr = LinkHierarchy(**record)
newr.save()
elif (act_table == 'oc_chronology'):
newr = Chronology(**record)
newr.save()
elif (act_table == 'oc_geodata'):
newr = Geodata(**record)
newr.save()
elif (act_table == 'oc_mediafiles'):
newr = Mediafile(**record)
newr.save()
elif (act_table == 'oc_documents'):
newr = OCdocument(**record)
newr.save()
elif (act_table == 'oc_persons'):
newr = Person(**record)
newr.save()
elif (act_table == 'oc_projects'):
newr = Project(**record)
newr.save()
elif (act_table == 'oc_strings'):
newr = OCstring(**record)
newr.save()
elif (act_table == 'oc_types'):
newr = OCtype(**record)
newr.save()
elif (act_table == 'oc_events'):
newr = Event(**record)
newr.save()
elif (act_table == 'oc_predicates'):
newr = Predicate(**record)
newr.save()
elif (act_table == 'oc_identifiers'):
newr = StableIdentifer(**record)
newr.save()
elif (act_table == 'oc_obsmetadata'):
newr = ObsMetadata(**record)
newr.save()
def load_csv(self, filename, after=0, add_path=False):
""" loads CSV dump from Merritt """
if add_path:
filename_path = os.path.join(settings.STATIC_ROOT,
self.DEFAULT_DIRECTORY,
filename)
else:
filename_path = filename
data = csv.reader(open(filename_path))
i = 0
for row in data:
manifest = False
if 'ark:/' in row[0]:
i += 0
if i >= after:
uuid = URImanagement.get_uuid_from_oc_uri(row[1])
if uuid is not False:
try:
manifest = Manifest.objects.get(uuid=uuid,
archived__isnull=True)
except Manifest.DoesNotExist:
manifest = False
if manifest is not False:
ok_new = True
try:
sid = StableIdentifer()
sid.stable_id = row[0].replace('ark:/', '')
sid.stable_type = 'ark'
sid.uuid = manifest.uuid
sid.project_uuid = manifest.project_uuid
sid.item_type = manifest.item_type
sid.save()
except:
ok_new = False
# note when the item was last archived
try:
manifest.archived = self.validate_date(row[3])
manifest.archived_save()
except:
manifest.archived = time.strftime('%Y-%m-%d %H:%M:%S')
manifest.archived_save()
if ok_new:
self.id_recorded += 1
print('Saved ids: ' + str(self.id_recorded))
def add_ids(self, ids):
""" Adds ids to the database
"""
if isinstance(ids, list):
for id_rec in ids:
id_and_type = self.parse_stable_id(id_rec['stable_id'])
manifest = False
uuid = URImanagement.get_uuid_from_oc_uri(id_rec['id'])
if uuid is not False and id_and_type is not False:
try:
manifest = Manifest.objects.get(uuid=uuid)
except Manifest.DoesNotExist:
manifest = False
if manifest is not False:
# we found the archived item in the manifest
# save the stable identifier in the database
ok_new = True
try:
sid = StableIdentifer()
sid.stable_id = id_and_type['id']
sid.stable_type = id_and_type['type']
sid.uuid = manifest.uuid
sid.project_uuid = manifest.project_uuid
sid.item_type = manifest.item_type
sid.save()
except:
ok_new = False
# note when the item was last archived
try:
manifest.archived = self.validate_date(
id_rec['archived'])
manifest.archived_save()
except:
manifest.archived = time.strftime('%Y-%m-%d %H:%M:%S')
manifest.archived_save()
if ok_new:
self.id_recorded += 1
return self.id_recorded
def save_oc_item_stable_id(self, oc_item, stable_id, stable_type='ark'):
""" saves stable_id for an oc_item object """
stable_id = stable_id.strip()
if 'doi:' in stable_id:
stable_type = 'doi'
stable_id = stable_id.replace('doi:', '')
elif 'ark:/' in stable_id:
stable_type = 'ark'
stable_id = stable_id.replace('ark:/', '')
print('Try to save ' + stable_type + ': ' + stable_id)
try:
ok = True
new_stable = StableIdentifer()
new_stable.stable_id = stable_id
new_stable.stable_type = stable_type
new_stable.uuid = oc_item.manifest.uuid
new_stable.project_uuid = oc_item.manifest.project_uuid
new_stable.item_type = oc_item.manifest.item_type
new_stable.save()
except:
ok = False
note = 'Identifier of type [' + stable_type + '] already exists.'
return ok
def save_oc_item_stable_id(self, oc_item, stable_id, stable_type='ark'):
""" saves stable_id for an oc_item object """
stable_id = stable_id.strip()
if 'doi:' in stable_id:
stable_type = 'doi'
stable_id = stable_id.replace('doi:', '')
elif 'ark:/' in stable_id:
stable_type = 'ark'
stable_id = stable_id.replace('ark:/', '')
print('Try to save ' + stable_type + ': ' + stable_id)
try:
ok = True
new_stable = StableIdentifer()
new_stable.stable_id = stable_id
new_stable.stable_type = stable_type
new_stable.uuid = oc_item.manifest.uuid
new_stable.project_uuid = oc_item.manifest.project_uuid
new_stable.item_type = oc_item.manifest.item_type
new_stable.save()
except:
ok = False
note = 'Identifier of type [' + stable_type + '] already exists.'
return ok
def add_ids(self, ids):
""" Adds ids to the database
"""
if isinstance(ids, list):
for id_rec in ids:
id_and_type = self.parse_stable_id(id_rec['stable_id'])
manifest = False
uuid = URImanagement.get_uuid_from_oc_uri(id_rec['id'])
if uuid is not False and id_and_type is not False:
try:
manifest = Manifest.objects.get(uuid=uuid)
except Manifest.DoesNotExist:
manifest = False
if manifest is not False:
# we found the archived item in the manifest
# save the stable identifier in the database
ok_new = True
try:
sid = StableIdentifer()
sid.stable_id = id_and_type['id']
sid.stable_type = id_and_type['type']
sid.uuid = manifest.uuid
sid.project_uuid = manifest.project_uuid
sid.item_type = manifest.item_type
sid.save()
except:
ok_new = False
# note when the item was last archived
try:
manifest.archived = self.validate_date(id_rec['archived'])
manifest.archived_save()
except:
manifest.archived = time.strftime('%Y-%m-%d %H:%M:%S')
manifest.archived_save()
if ok_new:
self.id_recorded += 1
return self.id_recorded