def commit_to_redis(self, vals):
self.log('Info', 'Committing Holdings updates to Redis...')
h_vals = vals.get('holdings', {})
er_vals = vals.get('eresources', {})
rev_handler = redisobjs.RedisObject('reverse_holdings_list', '0')
reverse_h_list = rev_handler.get()
for er_rec_num, lists in (h_vals or {}).items():
s = solr.Queryset().filter(record_number=er_rec_num)
try:
record = s[0]
except IndexError:
record = None
er_handler = redisobjs.RedisObject('eresource_holdings_list',
er_rec_num)
h_list = er_handler.get()
for h_rec_num in lists.get('delete', []):
h_index = h_list.index(h_rec_num)
del (h_list[h_index])
del (reverse_h_list[h_rec_num])
del (record.holdings[h_index])
for h_rec_num in lists.get('append', []):
h_list.append(h_rec_num)
reverse_h_list[h_rec_num] = er_rec_num
record.save()
er_handler.set(h_list)
rev_handler.set(reverse_h_list)
def final_callback(self, vals=None, status='success'):
vals = vals or {}
super(ItemsToSolr, self).final_callback(vals, status)
lcodes = list(vals.get('seen_lcodes', []))
total = len(lcodes)
msg = ('Building shelflist item manifest for {} location(s): {}'
''.format(total, ', '.join(lcodes)))
self.log('Info', msg)
for i, lcode in enumerate(lcodes):
manifest = self.indexes['Items'].get_location_manifest(lcode)
r = redisobjs.RedisObject(self.redis_shelflist_prefix, lcode)
r.set(manifest)
def commit_to_redis(self, vals):
self.log('Info', 'Committing EResource updates to Redis...')
rhl_obj = redisobjs.RedisObject('reverse_holdings_list', '0')
reverse_holdings = rhl_obj.get() or {}
# Update holdings for updated eresources
for ernum, h_list in vals.get('h_lists', {}).items():
redisobjs.RedisObject('eresource_holdings_list', ernum).set(h_list)
for hrnum in h_list:
reverse_holdings[hrnum] = ernum
# Delete holdings for deleted eresources
deletions = vals.get('deletions', [])
for ernum in deletions:
ehl_obj = redisobjs.RedisObject('eresource_holdings_list', ernum)
ehl_obj.conn.delete(ehl_obj.key)
if deletions:
for hrnum, ernum in reverse_holdings.items():
if ernum in deletions:
del (reverse_holdings[hrnum])
rhl_obj.set(reverse_holdings)
def final_callback(self, vals={}, status='success'):
if type(vals) is list:
vals = collapse_vals(vals)
self.log('Info', 'Committing updates to Solr and Redis...')
rev_handler = redisobjs.RedisObject('reverse_holdings_list', '0')
reverse_holdings_list = rev_handler.get() or {}
for er_rec_num, h_list in vals.get('h_lists', {}).iteritems():
er_handler = redisobjs.RedisObject('eresource_holdings_list',
er_rec_num)
er_handler.set(h_list)
for h_rec_num in h_list:
reverse_holdings_list[h_rec_num] = er_rec_num
#self.log('Info', 'Number of holdings in reverse_holdings_list: {}'
# .format(len(reverse_holdings_list.keys())))
rev_handler.set(reverse_holdings_list)
index = self.index_class()
index.commit(using=self.hs_conn)
def final_callback(self, vals={}, status='success'):
if type(vals) is list:
vals = collapse_vals(vals)
h_vals = vals.get('holdings', {})
er_vals = vals.get('eresources', {})
self.eresources_to_solr(self.instance.pk, self.export_filter,
self.export_type, self.options).final_callback(er_vals, status)
# commit changes to Redis and commit deletions to Solr
self.log('Info', 'Committing updates to Redis...')
rev_handler = redisobjs.RedisObject('reverse_holdings_list', '0')
reverse_h_list = rev_handler.get()
for er_rec_num, lists in h_vals.iteritems():
s = solr.Queryset().filter(record_number=er_rec_num)
try:
record = s[0]
except IndexError:
record = None
er_handler = redisobjs.RedisObject('eresource_holdings_list',
er_rec_num)
h_list = er_handler.get()
for h_rec_num in lists.get('delete', []):
h_index = h_list.index(h_rec_num)
del(h_list[h_index])
del(reverse_h_list[h_rec_num])
del(record.holdings[h_index])
for h_rec_num in lists.get('append', []):
h_list.append(h_rec_num)
reverse_h_list[h_rec_num] = er_rec_num
record.save()
er_handler.set(h_list)
rev_handler.set(reverse_h_list)
index = self.index_class()
index.commit(using=self.hs_conn)
def authenticate(self, request):
ret_val = None
username = request.META.get('HTTP_X_USERNAME', None)
timestamp = request.META.get('HTTP_X_TIMESTAMP', None)
client_signature = request.META.get('HTTP_AUTHORIZATION', 'Basic ')
client_signature = client_signature.split('Basic ')[1]
body = request.body
if username and timestamp and client_signature:
try:
api_user = models.APIUser.objects.get(user__username=username)
except models.APIUser.DoesNotExist:
raise exceptions.AuthenticationFailed('Incorrect username or '
'password.')
user_timestamp = ro.RedisObject('user_timestamp', username)
last_ts = user_timestamp.get() or 0
if last_ts >= float(timestamp):
raise exceptions.AuthenticationFailed('Timestamp invalid.')
secret = api_user.secret
user = api_user.user
hasher = hashlib.sha256('{}{}{}{}'.format(username, secret,
timestamp, body))
server_signature = hasher.hexdigest()
if server_signature != client_signature:
raise exceptions.AuthenticationFailed('Incorrect username or '
'password.')
else:
user_timestamp.set(float(timestamp))
ret_val = (user, None)
return ret_val
def export_records(self, records):
eresources, er_mapping = set(), {}
# First we loop through the holding records and determine which
# eresources need to be updated. er_mapping maps eresource rec
# nums to lists of holdings rec nums to update.
rev_handler = redisobjs.RedisObject('reverse_holdings_list', '0')
reverse_holdings_list = rev_handler.get() or {}
for h in records:
h_rec_num = h.record_metadata.get_iii_recnum(True)
old_er_rec_num = reverse_holdings_list.get(h_rec_num, None)
try:
er_record = h.resourcerecord_set.all()[0]
except IndexError:
er_record, er_rec_num = None, None
else:
er_rec_num = er_record.record_metadata.get_iii_recnum(True)
if old_er_rec_num and old_er_rec_num != er_rec_num:
# if the current attached er rec_num in Sierra is
# different than what's in Redis, then we need to
# delete this holding from the old er record.
old_h_data = er_mapping.get(old_er_rec_num, [])
old_h_data.append({
'delete': True,
'rec_num': h_rec_num,
'title': None
})
er_mapping[old_er_rec_num] = {
'er_record': None,
'holdings': old_h_data
}
if er_rec_num:
holdings = er_mapping.get(er_rec_num, {}).get('holdings', [])
try:
vf = h.bibrecord_set.all()[0].record_metadata\
.varfield_set.all()
except IndexError:
title = None
else:
title = helpers.get_varfield_vals(
vf,
't',
'245',
cm_kw_params={'subfields': 'a'},
content_method='display_field_content')
data = {'delete': False, 'title': title, 'rec_num': h_rec_num}
holdings.append(data)
er_mapping[er_rec_num] = {
'er_record': er_record,
'holdings': holdings
}
h_vals = {}
#self.log('Info', er_mapping)
for er_rec_num, entry in er_mapping.iteritems():
er_record, holdings = entry['er_record'], entry['holdings']
# if we've already indexed the eresource this holding is
# attached to, then we want to pull the record from Solr
# and make whatever changes to it rather than reindex the
# whole record and all attached holdings from scratch.
# Since export jobs get broken up and run in parallel, we
# want to hold off on actually committing to Solr and
# updating Redis until the callback runs.
s = solr.Queryset().filter(record_number=er_rec_num)
if s.count() > 0:
rec_queue = h_vals.get(er_rec_num, {})
rec_append_list = rec_queue.get('append', [])
rec_delete_list = rec_queue.get('delete', [])
record = s[0]
red = redisobjs.RedisObject('eresource_holdings_list',
er_rec_num)
red_h_list = red.get()
for data in holdings:
try:
red_h_index = red_h_list.index(data.get('rec_num'))
except AttributeError:
self.log('Info', '{}'.format(data.get('rec_num')))
except ValueError:
record.holdings.append(data.get('title'))
rec_append_list.append(data.get('rec_num'))
else:
if data.get('delete'):
# we wait until the final callback to
# delete anything from Solr, because that
# will mess up our holdings index number
rec_delete_list.append(data.get('rec_num'))
else:
record.holdings[red_h_index] = data.get('title')
record.save(commit=False)
rec_queue['append'] = rec_append_list
rec_queue['delete'] = rec_delete_list
h_vals[er_rec_num] = rec_queue
else:
# if we haven't indexed the record already, we'll add
# it using the Haystack indexer.
eresources.add(er_record)
if eresources:
eresources = list(eresources)
ret_er_vals = self.eresources_to_solr.export_records(eresources)
return {'holdings': h_vals, 'eresources': ret_er_vals}