Esempio n. 1
0
def create_id():
    # TODO: this is wasting an id
    cursor = connection.cursor()
    cursor.execute("SELECT nextval('reagent_id_seq')")
    row = cursor.fetchone();
    val = row[0]
    new_id = create_substance_id(val)
#     logger.info(str(('create_id', val, new_id)))
    cursor.execute("SELECT setval('reagent_id_seq', %s)", [val-1])
    return new_id
Esempio n. 2
0
def create_id():
    # TODO: There is no way to pass an argument to
    # to the default on a model, so this is why we have to grope around for the
    # sequence value.  The sequence here is used with the model idfield  
    try:
        cursor = connection.cursor()
        cursor.execute("SELECT nextval('db_substance_id_seq')")
        row = cursor.fetchone();
        val = row[0]
    
        new_id = create_substance_id(val)
        if(logger.isEnabledFor(logging.DEBUG)):
            logger.debug(str(('created_id', val, new_id)))
    #     if val > 1:
    #         cursor.execute("SELECT setval('db_substance_id_seq', %s)", [val-1])
        return new_id
    except Exception, e:
        logger.warn(str(('create_substance_id fails', e)))
        return None
def create_substance_ids(apps, schema_editor):
    connection = schema_editor.connection
    cursor = connection.cursor()
    count = 0
    batch_size = 1000
    logger.info('begin create_substance_ids')
    cursor.execute(
        'create temp table reagent_sub_ids as ' 
            "select nextval('substance_id_seq'), reagent_id " 
            "from reagent;")
    cursor.execute(
        'create temp table reagent_sub_ids1 '
            '(reagent_id integer, substance_id text )')
    cursor1 = connection.cursor()
    sql1 = 'insert into reagent_sub_ids1 values (%s, %s)'
    cursor.execute('Select * from reagent_sub_ids')
    cursor.arraysize = batch_size
    try:
        while True:
            rows = cursor.fetchmany()
            if not rows:
                break
            newrows = []
            for row in rows:
                newrows.append([row[1], create_substance_id(row[0])])
                count += 1
            cursor1.executemany(sql1, newrows)
            if count % 100000 == 0:
                logger.info('processed %d rows', count)

        logger.info('processed %d rows, now update reagent table', count)
        cursor.execute(
            'update reagent ' 
                'set substance_id=a.substance_id ' 
                'from reagent_sub_ids1 a '
                'where a.reagent_id=reagent.reagent_id')
    
    except Exception as e:
        logger.exception('create substance ids, ex after %d rows',count)
        raise
    
    logger.info('create substance ids, done after %d rows',count)
Esempio n. 4
0
def create_substance_ids(apps, schema_editor):
    connection = schema_editor.connection
    cursor = connection.cursor()
    count = 0
    batch_size = 1000
    logger.info('begin create_substance_ids')
    cursor.execute(
        'create temp table reagent_sub_ids as ' 
            "select nextval('substance_id_seq'), reagent_id " 
            "from reagent;")
    cursor.execute(
        'create temp table reagent_sub_ids1 '
            '(reagent_id integer, substance_id text )')
    cursor1 = connection.cursor()
    sql1 = 'insert into reagent_sub_ids1 values (%s, %s)'
    cursor.execute('Select * from reagent_sub_ids')
    cursor.arraysize = batch_size
    try:
        while True:
            rows = cursor.fetchmany()
            if not rows:
                break
            newrows = []
            for row in rows:
                newrows.append([row[1], create_substance_id(row[0])])
                count += 1
            cursor1.executemany(sql1, newrows)
            if count % 100000 == 0:
                logger.info('processed %d rows', count)

        logger.info('processed %d rows, now update reagent table', count)
        cursor.execute(
            'update reagent ' 
                'set substance_id=a.substance_id ' 
                'from reagent_sub_ids1 a '
                'where a.reagent_id=reagent.reagent_id')
    
    except Exception as e:
        logger.exception('create substance ids, ex after %d rows',count)
        raise
    
    logger.info('create substance ids, done after %d rows',count)