Beispiel #1
0
def mysql_to_rocket(field_type, mysql_value):
    if mysql_value == None:
        rocket_value = ""
    elif (field_type == TYPE_DATETIME or field_type == TYPE_TIMESTAMP):
        rocket_value = utils.to_iso(mysql_value)
    elif field_type == TYPE_KEY:
        rocket_value = mysql_to_rocket(TYPE_STR, mysql_value)
        if rocket_value[0] in '0123456789':
            # MYSQL ID
            rocket_value = '_%s' % rocket_value
        elif mysql_value[0] == '_':
            # APPENGINE ID
            rocket_value = rocket_value[1:]

    elif field_type == TYPE_REFERENCE:
        slash = mysql_value.find("/")
        if slash > 0:
            kind = mysql_value[:slash]
            key_name_or_id = mysql_to_rocket(TYPE_KEY, mysql_value[slash + 1:])
            rocket_value = "%s/%s" % (kind, key_name_or_id)
        else:
            logging.error("invalid reference value: %s" % mysql_value)
            rocket_value = ""
    elif field_type == TYPE_BLOB:
        rocket_value = base64.b64encode(mysql_value)
    else:
        rocket_value = (u'%s' % mysql_value).replace('|', '|').encode('utf-8')

    return rocket_value
Beispiel #2
0
def send_updates(kind, table_name, timestamp_field, table_key_field, send_fields, embedded_list_fields):
    cur = con.cursor()

    table = get_table_metadata(cur, table_name, timestamp_field, table_key_field, embedded_list_fields)

    if not table.fields.has_key(timestamp_field):
        logging.error('Error: table %s is missing timestamp field "%s"' % (table_name, timestamp_field))
        return

    if not table.fields.has_key(table_key_field):
        logging.error('Error: table %s is missing key field "%s"' % (table_name, table_key_field))
        return

    cur.execute("select current_timestamp")
    to_timestamp = cur.fetchone()[0] - timedelta(seconds=1) # -1 second to ensure there will be no more updates with that timestamp
    params = [to_timestamp]

    sql = "select %s from %s where %s < " % (', '.join(table.fields.keys()), table_name, timestamp_field) + """%s """
    if send_states.has_key(kind) and send_states[kind]:
        sql += "and " + timestamp_field + """ > %s """
        params.append(utils.from_iso(send_states[kind]))
        logging.info("send %s: from %s" % (kind, send_states[kind]))
    else:
        logging.info("send %s: from beginning" % (kind))

    sql += "order by %s " % timestamp_field

    offset = 0
    count = BATCH_SIZE
    while count == BATCH_SIZE:
        count = 0
        batch_sql = sql + " limit %d, %d" % (offset, BATCH_SIZE)
        cur.execute(batch_sql, params)
        intermediate_timestamp = None
        for row in cur.fetchall():
            count += 1

            key = None

            entity = {
            }

            i = 0
            for field_name, field_type in table.fields.items():

                field_value = row[i]

                if field_name == timestamp_field and field_value:
                    intermediate_timestamp = field_value - timedelta(seconds=1)
                    # do not send time stamp to avoid send/receive loop

                elif field_name == table_key_field:
                    key = field_value
                    entity[TYPE_KEY] = mysql_to_rocket(TYPE_KEY, field_value)

                elif field_type == TYPE_STR_LIST:
                    entity["*%s|%s" % (TYPE_STR, field_name)] = mysql_to_rocket(TYPE_STR, field_value)

                else:
                    if field_name.endswith("_ref"):
                        field_name = field_name[:len(field_name)-4]

                    if not send_fields or field_name in send_fields:
                        entity["%s|%s" % (field_type, field_name)] = mysql_to_rocket(field_type, field_value)

                i += 1


            if not key:
                logging.error('send %s: key field %s is empty' % (kind, table_key_field))
                continue

            # retrieve lists
            for field_name, field_type in table.list_fields.items():
                if not send_fields or field_name in send_fields:
                    cur.execute('select %s from %s_%s where %s = ' % (field_name, table_name, field_name, table_key_field) + """%s""", (key))

                    items = []
                    for item in cur.fetchall():
                        items.append(mysql_to_rocket(field_type, item[0]))

                    entity["*%s|%s" % (field_type, field_name)] = '|'.join(items)

            logging.debug('send %s: key=%s' % (kind, key))

            for attempt in range(3): # try three times
                try:
                    send_row(kind, key, entity)
                    break
                except:
                    logging.exception('send %s: key=%s, attempt #%d failed' % (kind, key, attempt + 1))
            else:
                # if all retries failed - rollback and return
                con.rollback()
                return

        logging.info('send %s: batch end, count=%d, offset=%d' % (kind, count, offset))
        offset += count

        if intermediate_timestamp:
            intermediate_timestamp = utils.to_iso(intermediate_timestamp)
            write_send_state(cur, kind, intermediate_timestamp)
            con.commit()
            send_states[kind] = intermediate_timestamp

    to_timestamp = utils.to_iso(to_timestamp)
    write_send_state(cur, kind, to_timestamp)
    con.commit()
    send_states[kind] = to_timestamp

    cur.close()

    return count > 0 or offset > 0