def mysql_to_rocket(field_type, mysql_value):
if mysql_value == None:
rocket_value = ""
elif (field_type == TYPE_DATETIME or field_type == TYPE_TIMESTAMP):
rocket_value = utils.to_iso(mysql_value)
elif field_type == TYPE_KEY:
rocket_value = mysql_to_rocket(TYPE_STR, mysql_value)
if rocket_value[0] in '0123456789':
# MYSQL ID
rocket_value = '_%s' % rocket_value
elif mysql_value[0] == '_':
# APPENGINE ID
rocket_value = rocket_value[1:]
elif field_type == TYPE_REFERENCE:
slash = mysql_value.find("/")
if slash > 0:
kind = mysql_value[:slash]
key_name_or_id = mysql_to_rocket(TYPE_KEY, mysql_value[slash + 1:])
rocket_value = "%s/%s" % (kind, key_name_or_id)
else:
logging.error("invalid reference value: %s" % mysql_value)
rocket_value = ""
elif field_type == TYPE_BLOB:
rocket_value = base64.b64encode(mysql_value)
else:
rocket_value = (u'%s' % mysql_value).replace('|', '|').encode('utf-8')
return rocket_value
def send_updates(kind, table_name, timestamp_field, table_key_field, send_fields, embedded_list_fields):
cur = con.cursor()
table = get_table_metadata(cur, table_name, timestamp_field, table_key_field, embedded_list_fields)
if not table.fields.has_key(timestamp_field):
logging.error('Error: table %s is missing timestamp field "%s"' % (table_name, timestamp_field))
return
if not table.fields.has_key(table_key_field):
logging.error('Error: table %s is missing key field "%s"' % (table_name, table_key_field))
return
cur.execute("select current_timestamp")
to_timestamp = cur.fetchone()[0] - timedelta(seconds=1) # -1 second to ensure there will be no more updates with that timestamp
params = [to_timestamp]
sql = "select %s from %s where %s < " % (', '.join(table.fields.keys()), table_name, timestamp_field) + """%s """
if send_states.has_key(kind) and send_states[kind]:
sql += "and " + timestamp_field + """ > %s """
params.append(utils.from_iso(send_states[kind]))
logging.info("send %s: from %s" % (kind, send_states[kind]))
else:
logging.info("send %s: from beginning" % (kind))
sql += "order by %s " % timestamp_field
offset = 0
count = BATCH_SIZE
while count == BATCH_SIZE:
count = 0
batch_sql = sql + " limit %d, %d" % (offset, BATCH_SIZE)
cur.execute(batch_sql, params)
intermediate_timestamp = None
for row in cur.fetchall():
count += 1
key = None
entity = {
}
i = 0
for field_name, field_type in table.fields.items():
field_value = row[i]
if field_name == timestamp_field and field_value:
intermediate_timestamp = field_value - timedelta(seconds=1)
# do not send time stamp to avoid send/receive loop
elif field_name == table_key_field:
key = field_value
entity[TYPE_KEY] = mysql_to_rocket(TYPE_KEY, field_value)
elif field_type == TYPE_STR_LIST:
entity["*%s|%s" % (TYPE_STR, field_name)] = mysql_to_rocket(TYPE_STR, field_value)
else:
if field_name.endswith("_ref"):
field_name = field_name[:len(field_name)-4]
if not send_fields or field_name in send_fields:
entity["%s|%s" % (field_type, field_name)] = mysql_to_rocket(field_type, field_value)
i += 1
if not key:
logging.error('send %s: key field %s is empty' % (kind, table_key_field))
continue
# retrieve lists
for field_name, field_type in table.list_fields.items():
if not send_fields or field_name in send_fields:
cur.execute('select %s from %s_%s where %s = ' % (field_name, table_name, field_name, table_key_field) + """%s""", (key))
items = []
for item in cur.fetchall():
items.append(mysql_to_rocket(field_type, item[0]))
entity["*%s|%s" % (field_type, field_name)] = '|'.join(items)
logging.debug('send %s: key=%s' % (kind, key))
for attempt in range(3): # try three times
try:
send_row(kind, key, entity)
break
except:
logging.exception('send %s: key=%s, attempt #%d failed' % (kind, key, attempt + 1))
else:
# if all retries failed - rollback and return
con.rollback()
return
logging.info('send %s: batch end, count=%d, offset=%d' % (kind, count, offset))
offset += count
if intermediate_timestamp:
intermediate_timestamp = utils.to_iso(intermediate_timestamp)
write_send_state(cur, kind, intermediate_timestamp)
con.commit()
send_states[kind] = intermediate_timestamp
to_timestamp = utils.to_iso(to_timestamp)
write_send_state(cur, kind, to_timestamp)
con.commit()
send_states[kind] = to_timestamp
cur.close()
return count > 0 or offset > 0