def denorm_instance(payload, cursor=None):
logging.info('[cursor.denorm_instance] payload %s, cursor %s' %
(payload, cursor))
data = json.loads(payload)
source_model = util.get_model_by_name(data['source_model'])
target_model = util.get_model_by_name(data['target_model'])
related_field_name = data['related_field']
fields = data['fields']
queryset = target_model.objects.filter(
**{related_field_name + '_id': data['instance_id']})
if cursor:
queryset = set_cursor(queryset, cursor)
results = queryset[0:ITEMS_PER_TASK]
cursor = get_cursor(results)
# TODO: batch save
for item in results:
#print('[denorm_instance] denorm target instance %s' % item)
item._denorm_values = fields # provide denorm values directly so that pre_save signal receiver does not lookup related field
item.save()
if len(results) == ITEMS_PER_TASK:
# there are likely more items
logging.info('[denorm_instance] queue task with cursor %s' % cursor)
deferred.defer(denorm_instance,
payload,
cursor,
_queue=data['queue_name'])
def denorm_instance(payload, cursor=None):
logging.info('[cursor.denorm_instance] payload %s, cursor %s' % (payload, cursor))
data = json.loads(payload)
source_model = util.get_model_by_name(data['source_model'])
target_model = util.get_model_by_name(data['target_model'])
related_field_name = data['related_field']
fields = data['fields']
queryset = target_model.objects.filter(**{related_field_name+'_id': data['instance_id']})
if cursor:
queryset = set_cursor(queryset, cursor)
results = queryset[0:ITEMS_PER_TASK]
cursor = get_cursor(results)
# TODO: batch save
for item in results:
#print('[denorm_instance] denorm target instance %s' % item)
item._denorm_values = fields # provide denorm values directly so that pre_save signal receiver does not lookup related field
item.save()
if len(results) == ITEMS_PER_TASK:
# there are likely more items
logging.info('[denorm_instance] queue task with cursor %s' % cursor)
deferred.defer(denorm_instance, payload, cursor, _queue=data['queue_name'])
def denorm_instance(payload):
logging.info('[map_reduce.denorm_instance] payload %s' % json.dumps(payload))
source_model = util.get_model_by_name(payload['source_model'])
target_model = util.get_model_by_name(payload['target_model'])
related_field_name = payload['related_field']
fields = payload['fields']
storage = payload['storage']
if storage == 'scalar':
related_field_name_filter = related_field_name+'_id'
denorm_values = fields
else:
assert(storage == 'shared_dict')
# will look up source primary key in target's list field
related_field_name_filter = Inflector().pluralize(related_field_name)
denorm_values = {
'denorm_data': {
related_field_name_filter: {
payload['instance_id']: fields
}
}
}
pipeline = MapperPipeline(
'denorm-target-%s-source-%s-instance-%s-at-%s' % (payload['target_model'], payload['source_model'], payload['instance_id'], now().isoformat()),
handler_spec = util.convert_func_to_string(denorm_entity_mapper),
input_reader_spec = 'djangoappengine.mapreduce.input_readers.DjangoModelInputReader', # FIXME: should be self-contained
output_writer_spec = util.convert_func_to_string(NullOutputWriter),
params = {
# _convert_model_to_string is a different encoding than util.get_model_name, but we have to use because
# we use djangoappengine's input reader
'entity_kind': _convert_model_to_string(target_model),
'queue_name': payload['queue_name'],
'filters': [
[related_field_name_filter, '=', payload['instance_id']],
],
'denorm_values': denorm_values,
#'storage': storage,
},
shards = payload['shards']
)
pipeline.start(queue_name=payload['queue_name'])
pipeline_id = pipeline.pipeline_id
logging.info('[map_reduce.denorm_instance] pipeline_id = %s' % str(pipeline_id))