def proccess_pokemons(self, config, pokemons): if not pokemons: raise Exception("No pokemons to process") csv_rows = [] for pokemon in pokemons: pokemonInfo = pokemon_services.get_pokemon(pokemon["name"]) values = [] for fieldPath in config["csv"]["fields"]: try: values.append(self.filter_fields(fieldPath, pokemonInfo)) except Exception as e: logger.error(str(e)) logger.error( f"fieldPath = {fieldPath}, pokemonInfo = {pokemonInfo}" ) raise Exception("Error in filter fields") csv_rows.append(values) # measure_memory.measure() garbage_collect() sleep_execution(INTERVAL_REQUISITIONS) self.write_rows(csv_rows)
def upgrade(): from inbox.server.config import load_config load_config() from inbox.sqlalchemy.util import generate_public_id from inbox.server.models import session_scope # These all inherit HasPublicID from inbox.server.models.tables.base import ( Account, Block, Contact, Message, Namespace, SharedFolder, Thread, User, UserSession, HasPublicID) classes = [ Account, Block, Contact, Message, Namespace, SharedFolder, Thread, User, UserSession] for c in classes: assert issubclass(c, HasPublicID) print '[{0}] adding public_id column... '.format(c.__tablename__), sys.stdout.flush() op.add_column(c.__tablename__, sa.Column( 'public_id', mysql.BINARY(16), nullable=False)) print 'adding index... ', op.create_index( 'ix_{0}_public_id'.format(c.__tablename__), c.__tablename__, ['public_id'], unique=False) print 'Done!' sys.stdout.flush() print 'Finished adding columns. \nNow generating public_ids' with session_scope() as db_session: count = 0 for c in classes: garbage_collect() print '[{0}] Loading rows. '.format(c.__name__), sys.stdout.flush() print 'Generating public_ids', sys.stdout.flush() for r in db_session.query(c).yield_per(chunk_size): count += 1 r.public_id = generate_public_id() if not count % chunk_size: sys.stdout.write('.') sys.stdout.flush() db_session.commit() garbage_collect() sys.stdout.write(' Saving. '.format(c.__name__)), # sys.stdout.flush() sys.stdout.flush() db_session.commit() sys.stdout.write('Done!\n') sys.stdout.flush() print '\nUpdgraded OK!\n'
def write_rows(self, rows): if not rows: raise Exception('No rows to write') with open(self.local_path, 'a', newline=NEWLINE) as csvFile: csv_writer = csv.writer(csvFile, **CSV_WRITER) csv_writer.writerows(rows) garbage_collect()
def upgrade(): op.add_column("transaction", sa.Column("public_id", mysql.BINARY(16), nullable=True)) op.add_column( "transaction", sa.Column("object_public_id", sa.String(length=191), nullable=True), ) op.create_index("ix_transaction_public_id", "transaction", ["public_id"], unique=False) # TODO(emfree) reflect from inbox.ignition import main_engine from inbox.models.session import session_scope from inbox.sqlalchemy_ext.util import b36_to_bin, generate_public_id engine = main_engine(pool_size=1, max_overflow=0) Base = declarative_base() Base.metadata.reflect(engine) class Transaction(Base): __table__ = Base.metadata.tables["transaction"] with session_scope(versioned=False) as db_session: count = 0 (num_transactions, ) = db_session.query(sa.func.max( Transaction.id)).one() print("Adding public ids to {} transactions".format(num_transactions)) for pointer in range(0, num_transactions + 1, 500): for entry in db_session.query(Transaction).filter( Transaction.id >= pointer, Transaction.id < pointer + 500): entry.public_id = b36_to_bin(generate_public_id()) count += 1 if not count % 500: sys.stdout.write(".") sys.stdout.flush() db_session.commit() garbage_collect() op.alter_column("transaction", "public_id", existing_type=mysql.BINARY(16), nullable=False) op.add_column( "transaction", sa.Column("public_snapshot", sa.Text(length=4194304), nullable=True), ) op.add_column( "transaction", sa.Column("private_snapshot", sa.Text(length=4194304), nullable=True), ) op.drop_column("transaction", u"additional_data")
def upgrade(): op.add_column('transaction', sa.Column('public_id', mysql.BINARY(16), nullable=True)) op.add_column( 'transaction', sa.Column('object_public_id', sa.String(length=191), nullable=True)) op.create_index('ix_transaction_public_id', 'transaction', ['public_id'], unique=False) from inbox.sqlalchemy_ext.util import generate_public_id, b36_to_bin # TODO(emfree) reflect from inbox.models.session import session_scope from inbox.ignition import main_engine engine = main_engine(pool_size=1, max_overflow=0) Base = declarative_base() Base.metadata.reflect(engine) class Transaction(Base): __table__ = Base.metadata.tables['transaction'] with session_scope(versioned=False, ignore_soft_deletes=False) as db_session: count = 0 num_transactions, = db_session.query(sa.func.max(Transaction.id)).one() print 'Adding public ids to {} transactions'.format(num_transactions) for pointer in range(0, num_transactions + 1, 500): for entry in db_session.query(Transaction).filter( Transaction.id >= pointer, Transaction.id < pointer + 500): entry.public_id = b36_to_bin(generate_public_id()) count += 1 if not count % 500: sys.stdout.write('.') sys.stdout.flush() db_session.commit() garbage_collect() op.alter_column('transaction', 'public_id', existing_type=mysql.BINARY(16), nullable=False) op.add_column( 'transaction', sa.Column('public_snapshot', sa.Text(length=4194304), nullable=True)) op.add_column( 'transaction', sa.Column('private_snapshot', sa.Text(length=4194304), nullable=True)) op.drop_column('transaction', u'additional_data')
def commit_uids(db_session, log, new_uids): new_messages = [item.message for item in new_uids] # Save message part blobs before committing changes to db. for msg in new_messages: threads = [ Greenlet.spawn(log_uncaught_errors(part.save, log), part._data) for part in msg.parts if hasattr(part, '_data') ] # Fatally abort if part saves error out. Messages in this # chunk will be retried when the sync is restarted. gevent_check_join(log, threads, "Could not save message parts to blob store!") # clear data to save memory for part in msg.parts: part._data = None garbage_collect() try: log.info("Committing {0} UIDs".format(len(new_uids))) db_session.add_all(new_uids) db_session.commit() except DataError as e: db_session.rollback() log.error("Issue inserting new UIDs into database. " "This probably means that an object's property is " "malformed or way too long, etc.") for uid in new_uids: log.error(uid) import inspect from pprint import pformat log.error(inspect.getmembers(uid)) try: log.error(pformat(uid.__dict__, indent=2)) except AttributeError: pass for part in uid.message.parts: log.error(inspect.getmembers(part)) try: log.error(pformat(part.__dict__, indent=2)) except AttributeError: pass raise e
def commit_uids(db_session, log, new_uids): new_messages = [item.message for item in new_uids] # Save message part blobs before committing changes to db. for msg in new_messages: threads = [Greenlet.spawn(retry_with_logging, lambda: part.save(part._data), log) for part in msg.parts if hasattr(part, '_data')] # Fatally abort if part saves error out. Messages in this # chunk will be retried when the sync is restarted. gevent_check_join(log, threads, "Could not save message parts to blob store!") # clear data to save memory for part in msg.parts: part._data = None garbage_collect() try: log.info("Committing {0} UIDs".format(len(new_uids))) db_session.add_all(new_uids) db_session.commit() except DataError as e: db_session.rollback() log.error("Issue inserting new UIDs into database. " "This probably means that an object's property is " "malformed or way too long, etc.") for uid in new_uids: log.error(uid) import inspect from pprint import pformat log.error(inspect.getmembers(uid)) try: log.error(pformat(uid.__dict__, indent=2)) except AttributeError: pass for part in uid.message.parts: log.error(inspect.getmembers(part)) try: log.error(pformat(part.__dict__, indent=2)) except AttributeError: pass raise e
def upgrade(): op.add_column('transaction', sa.Column('public_id', mysql.BINARY(16), nullable=True)) op.add_column('transaction', sa.Column('object_public_id', sa.String(length=191), nullable=True)) op.create_index('ix_transaction_public_id', 'transaction', ['public_id'], unique=False) from inbox.sqlalchemy_ext.util import generate_public_id, b36_to_bin # TODO(emfree) reflect from inbox.models.session import session_scope from inbox.ignition import main_engine engine = main_engine(pool_size=1, max_overflow=0) Base = declarative_base() Base.metadata.reflect(engine) class Transaction(Base): __table__ = Base.metadata.tables['transaction'] with session_scope(versioned=False) as db_session: count = 0 num_transactions, = db_session.query(sa.func.max(Transaction.id)).one() print 'Adding public ids to {} transactions'.format(num_transactions) for pointer in range(0, num_transactions + 1, 500): for entry in db_session.query(Transaction).filter( Transaction.id >= pointer, Transaction.id < pointer + 500): entry.public_id = b36_to_bin(generate_public_id()) count += 1 if not count % 500: sys.stdout.write('.') sys.stdout.flush() db_session.commit() garbage_collect() op.alter_column('transaction', 'public_id', existing_type=mysql.BINARY(16), nullable=False) op.add_column('transaction', sa.Column('public_snapshot', sa.Text(length=4194304), nullable=True)) op.add_column('transaction', sa.Column('private_snapshot', sa.Text(length=4194304), nullable=True)) op.drop_column('transaction', u'additional_data')
def get_run_metrics( self, test_result: TestResult, run_metrics: RunMetrics ) -> MaybeMetricValuesForSingleIteration: """Discards the trace when done to avoid using too much memory.""" already_had = test_result in self.path_to_file # TODO: may need join info res = map_ok( self.get(test_result, need_mechanisms_and_reasons=False, need_join_info=False), lambda trace: map_to_mapping(run_metrics, trace.metric), ) if not already_had: del self.path_to_file[test_result] # Ensure the trace is freed. # Without this, Python won't do a collection, and C# will run out of memory. # C# garbage collection isn't enough since Python needs to do GC to release the trace. garbage_collect() return res
def commit_uids(db_session, log, new_imapuids): new_messages = [item.message for item in new_imapuids] # Save message part blobs before committing changes to db. for msg in new_messages: threads = [Greenlet.spawn(part.save, part._data) \ for part in msg.parts if hasattr(part, '_data')] # Fatally abort if part saves error out. Messages in this # chunk will be retried when the sync is restarted. gevent_check_join(log, threads, "Could not save message parts to blob store!") # clear data to save memory for part in msg.parts: part._data = None garbage_collect() db_session.add_all(new_imapuids) db_session.commit()
def upgrade(): # These all inherit HasPublicID from inbox.models import ( Account, Block, Contact, HasPublicID, Message, Namespace, SharedFolder, Thread, User, UserSession, ) from inbox.models.session import session_scope from inbox.sqlalchemy_ext.util import generate_public_id classes = [ Account, Block, Contact, Message, Namespace, SharedFolder, Thread, User, UserSession, ] for c in classes: assert issubclass(c, HasPublicID) print "[{0}] adding public_id column... ".format(c.__tablename__), sys.stdout.flush() op.add_column(c.__tablename__, sa.Column("public_id", mysql.BINARY(16), nullable=False)) print "adding index... ", op.create_index( "ix_{0}_public_id".format(c.__tablename__), c.__tablename__, ["public_id"], unique=False, ) print "Done!" sys.stdout.flush() print "Finished adding columns. \nNow generating public_ids" with session_scope() as db_session: count = 0 for c in classes: garbage_collect() print "[{0}] Loading rows. ".format(c.__name__), sys.stdout.flush() print "Generating public_ids", sys.stdout.flush() for r in db_session.query(c).yield_per(chunk_size): count += 1 r.public_id = generate_public_id() if not count % chunk_size: sys.stdout.write(".") sys.stdout.flush() db_session.commit() garbage_collect() sys.stdout.write(" Saving. ".format(c.__name__)), # sys.stdout.flush() sys.stdout.flush() db_session.commit() sys.stdout.write("Done!\n") sys.stdout.flush() print "\nUpdgraded OK!\n"