def test_json(self): b = BibRecord(bibcode=u'\u01b5') cls, data = b.dump() x = json.dumps(b.__json__()) z = json.loads(x) data2 = base64.b64decode(z['__adsmsg__'][1]) self.assertEqual(repr(data), repr(data2)) b1 = Msg.loads(cls, data) b2 = Msg.loads(cls, data2) self.assertEqual(b1.bibcode, u'\u01b5') self.assertEqual(b2.bibcode, u'\u01b5')
def test_serializer(self): b = BibRecord(bibcode='bibcode') cls, data = b.dump() self.assertEqual('adsmsg.bibrecord.BibRecord', cls) self.assertEqual('\n\x07bibcode', data) b2 = Msg.loads(cls, data) self.assertEqual(b2.bibcode, b.bibcode)
def test_higher_char(self): b = BibRecord(bibcode=u'\u01b5') cls, data = b.dump() self.assertEqual('adsmsg.bibrecord.BibRecord', cls) self.assertEqual('\n\x02\xc6\xb5', data) b2 = Msg.loads(cls, data) self.assertEqual(b2.bibcode, b.bibcode)
def task_update_record(msg): """Receives payload to update the record. @param msg: protobuff that contains at minimum - bibcode - and specific payload """ logger.debug('Updating record: %s', msg) status = app.get_msg_status(msg) type = app.get_msg_type(msg) bibcodes = [] if status == 'deleted': if type == 'metadata': task_delete_documents(msg.bibcode) elif type == 'nonbib_records': for m in msg.nonbib_records: # TODO: this is very ugly, we are repeating ourselves... bibcodes.append(m.bibcode) record = app.update_storage(m.bibcode, 'nonbib_data', None) if record: logger.debug('Deleted %s, result: %s', type, record) elif type == 'metrics_records': for m in msg.metrics_records: bibcodes.append(m.bibcode) record = app.update_storage(m.bibcode, 'metrics', None) if record: logger.debug('Deleted %s, result: %s', type, record) else: bibcodes.append(msg.bibcode) record = app.update_storage(msg.bibcode, type, None) if record: logger.debug('Deleted %s, result: %s', type, record) elif status == 'active': # save into a database # passed msg may contain details on one bibcode or a list of bibcodes if type == 'nonbib_records': for m in msg.nonbib_records: m = Msg(m, None, None) # m is a raw protobuf, TODO: return proper instance from .nonbib_records bibcodes.append(m.bibcode) record = app.update_storage(m.bibcode, 'nonbib_data', m.toJSON()) if record: logger.debug('Saved record from list: %s', record) elif type == 'metrics_records': for m in msg.metrics_records: m = Msg(m, None, None) bibcodes.append(m.bibcode) record = app.update_storage(m.bibcode, 'metrics', m.toJSON(including_default_value_fields=True)) if record: logger.debug('Saved record from list: %s', record) elif type =='augment': bibcodes.append(msg.bibcode) record = app.update_storage(msg.bibcode, 'augment', msg.toJSON(including_default_value_fields=True)) if record: logger.debug('Saved augment message: %s', msg) else: # here when record has a single bibcode bibcodes.append(msg.bibcode) record = app.update_storage(msg.bibcode, type, msg.toJSON()) if record: logger.debug('Saved record: %s', record) if type == 'metadata': # with new bib data we request to augment the affiliation # that pipeline will eventually respond with a msg to task_update_record logger.info('requesting affilation augmentation for %s', msg.bibcode) app.request_aff_augment(msg.bibcode) else: logger.error('Received a message with unclear status: %s', msg)
def adsmsg_converter(dct): if '__adsmsg__' in dct: cls, data = dct['__adsmsg__'] return Msg.loads(cls, base64.b64decode(data)) # ('class.name', 'binary data.....') return dct