def create_id(): # TODO: this is wasting an id cursor = connection.cursor() cursor.execute("SELECT nextval('reagent_id_seq')") row = cursor.fetchone(); val = row[0] new_id = create_substance_id(val) # logger.info(str(('create_id', val, new_id))) cursor.execute("SELECT setval('reagent_id_seq', %s)", [val-1]) return new_id
def create_id(): # TODO: There is no way to pass an argument to # to the default on a model, so this is why we have to grope around for the # sequence value. The sequence here is used with the model idfield try: cursor = connection.cursor() cursor.execute("SELECT nextval('db_substance_id_seq')") row = cursor.fetchone(); val = row[0] new_id = create_substance_id(val) if(logger.isEnabledFor(logging.DEBUG)): logger.debug(str(('created_id', val, new_id))) # if val > 1: # cursor.execute("SELECT setval('db_substance_id_seq', %s)", [val-1]) return new_id except Exception, e: logger.warn(str(('create_substance_id fails', e))) return None
def create_substance_ids(apps, schema_editor): connection = schema_editor.connection cursor = connection.cursor() count = 0 batch_size = 1000 logger.info('begin create_substance_ids') cursor.execute( 'create temp table reagent_sub_ids as ' "select nextval('substance_id_seq'), reagent_id " "from reagent;") cursor.execute( 'create temp table reagent_sub_ids1 ' '(reagent_id integer, substance_id text )') cursor1 = connection.cursor() sql1 = 'insert into reagent_sub_ids1 values (%s, %s)' cursor.execute('Select * from reagent_sub_ids') cursor.arraysize = batch_size try: while True: rows = cursor.fetchmany() if not rows: break newrows = [] for row in rows: newrows.append([row[1], create_substance_id(row[0])]) count += 1 cursor1.executemany(sql1, newrows) if count % 100000 == 0: logger.info('processed %d rows', count) logger.info('processed %d rows, now update reagent table', count) cursor.execute( 'update reagent ' 'set substance_id=a.substance_id ' 'from reagent_sub_ids1 a ' 'where a.reagent_id=reagent.reagent_id') except Exception as e: logger.exception('create substance ids, ex after %d rows',count) raise logger.info('create substance ids, done after %d rows',count)