def clean_orphan_tags(commit=True): """Search for unused MediaTags and delete them""" q1 = Session.query(Tag).outerjoin(MediaTag).filter(MediaTag.id==None) for t in q1: Session.delete(t) # The "let the db do all the work" version: # q1 = Session.query(Tag.id).outerjoin(MediaTag).filter(MediaTag.id==None) # q2 = Session.query(Tag).filter(Tag.id.in_(q1)) # q2.delete(synchronize_session = False) if commit: Session.commit()
def clean_orphan_tags(): q1 = Session.query(Tag).outerjoin(MediaTag).filter(MediaTag.id==None) for t in q1: Session.delete(t) # The "let the db do all the work" version: # q1 = Session.query(Tag.id).outerjoin(MediaTag).filter(MediaTag.id==None) # q2 = Session.query(Tag).filter(Tag.id.in_(q1)) # q2.delete(synchronize_session = False) Session.commit()
def get_test_app(dump_old_app=True): suicide_if_bad_celery_environ() # Make sure we've turned on testing testing._activate_testing() # Leave this imported as it sets up celery. from mediagoblin.init.celery import from_tests global MGOBLIN_APP # Just return the old app if that exists and it's okay to set up # and return if MGOBLIN_APP and not dump_old_app: return MGOBLIN_APP Session.rollback() Session.remove() # Remove and reinstall user_dev directories if os.path.exists(TEST_USER_DEV): shutil.rmtree(TEST_USER_DEV) for directory in USER_DEV_DIRECTORIES_TO_SETUP: full_dir = os.path.join(TEST_USER_DEV, directory) os.makedirs(full_dir) # Get app config global_config, validation_result = read_mediagoblin_config(TEST_APP_CONFIG) app_config = global_config['mediagoblin'] # Run database setup/migrations run_dbupdate(app_config, global_config) # setup app and return test_app = loadapp( 'config:' + TEST_SERVER_CONFIG) # Re-setup celery setup_celery_app(app_config, global_config) # Insert the TestingMeddleware, which can do some # sanity checks on every request/response. # Doing it this way is probably not the cleanest way. # We'll fix it, when we have plugins! mg_globals.app.meddleware.insert(0, TestingMeddleware(mg_globals.app)) app = TestApp(test_app) MGOBLIN_APP = app return app
def check_media_slug_used(dummy_db, uploader_id, slug, ignore_m_id): filt = (MediaEntry.uploader == uploader_id) \ & (MediaEntry.slug == slug) if ignore_m_id is not None: filt = filt & (MediaEntry.id != ignore_m_id) does_exist = Session.query(MediaEntry.id).filter(filt).first() is not None return does_exist
def check_collection_slug_used(dummy_db, creator_id, slug, ignore_c_id): filt = (Collection.creator == creator_id) \ & (Collection.slug == slug) if ignore_c_id is not None: filt = filt & (Collection.id != ignore_c_id) does_exist = Session.query(Collection.id).filter(filt).first() is not None return does_exist
def run_conversion(config_name): global_config, app_config = setup_global_and_app_config(config_name) sql_conn, sql_db = sql_connect(app_config) mk_conn, mk_db = mongo_connect(app_config) Base_v0.metadata.create_all(sql_db.engine) for title, func in convert_call_list: print_header(title) func(mk_db) Session.remove() for title, func in sql_call_list: print_header(title) func(sql_db) Session.remove()
def media_data_init(self, **kwargs): """ Initialize or update the contents of a media entry's media_data row """ session = Session() media_data = session.query(self.media_data_table).filter_by( media_entry=self.id).first() # No media data, so actually add a new one if media_data is None: media_data = self.media_data_table( media_entry=self.id, **kwargs) session.add(media_data) # Update old media data else: for field, value in kwargs.iteritems(): setattr(media_data, field, value)
def fixture_add_user(username=u'chris', password='******', active_user=True): test_user = mg_globals.database.User() test_user.username = username test_user.email = username + u'@example.com' if password is not None: test_user.pw_hash = bcrypt_gen_password_hash(password) if active_user: test_user.email_verified = True test_user.status = u'active' test_user.save() # Reload test_user = mg_globals.database.User.find_one({'username': username}) # ... and detach from session: from mediagoblin.db.sql.base import Session Session.expunge(test_user) return test_user
def convert_add_migration_versions(dummy_sql_db): session = Session() for name in chain(("__main__",), imap(lambda e: e[0], media_types_tables)): print "\tAdding %s" % (name,) m = MigrationData(name=unicode(name), version=0) session.add(m) session.commit() session.close()
def convert_video(mk_db): session = Session() for media in mk_db.MediaEntry.find( {'media_type': 'mediagoblin.media_types.video'}).sort('created'): media_data_row = VideoData(**media.media_data) media_data_row.media_entry = obj_id_table[media['_id']] session.add(media_data_row) session.commit() session.close()
def convert_users(mk_db): session = Session() for entry in mk_db.User.find().sort('created'): print entry.username new_entry = User() copy_attrs(entry, new_entry, ('username', 'email', 'created', 'pw_hash', 'email_verified', 'status', 'verification_key', 'is_admin', 'url', 'bio', 'fp_verification_key', 'fp_token_expire',)) # new_entry.fp_verification_expire = entry.fp_token_expire session.add(new_entry) session.flush() add_obj_ids(entry, new_entry) session.commit() session.close()
def convert_image(mk_db): session = Session() for media in mk_db.MediaEntry.find( {'media_type': 'mediagoblin.media_types.image'}).sort('created'): media_data = copy(media.media_data) if len(media_data): media_data_row = ImageData(**media_data) media_data_row.media_entry = obj_id_table[media['_id']] session.add(media_data_row) session.commit() session.close()
def convert_media_comments(mk_db): session = Session() for entry in mk_db.MediaComment.find().sort('created'): print repr(entry.content) new_entry = MediaComment() copy_attrs(entry, new_entry, ('created', 'content',)) try: copy_reference_attr(entry, new_entry, "media_entry") copy_reference_attr(entry, new_entry, "author") except KeyError as e: print('KeyError in convert_media_comments(): {0}'.format(e)) else: session.add(new_entry) session.flush() add_obj_ids(entry, new_entry) session.commit() session.close()
def reset_after_request(self): Session.rollback() Session.remove()
def media_data(self): session = Session() return session.query(self.media_data_table).filter_by( media_entry=self.id).first()
def convert_media_tags(mk_db): session = Session() session.autoflush = False for media in mk_db.MediaEntry.find().sort('created'): print repr(media.title) for otag in media.tags: print " ", repr((otag["slug"], otag["name"])) nslug = session.query(Tag).filter_by(slug=otag["slug"]).first() print " ", repr(nslug) if nslug is None: nslug = Tag(slug=otag["slug"]) session.add(nslug) session.flush() print " ", repr(nslug), nslug.id ntag = MediaTag() ntag.tag = nslug.id ntag.name = otag["name"] ntag.media_entry = obj_id_table[media._id] session.add(ntag) session.commit() session.close()
def cleanup_sql_tables(sql_db): for mt, table_list in media_types_tables: session = Session() count = session.query(MediaEntry.media_type). \ filter_by(media_type=unicode(mt)).count() print " %s: %d entries" % (mt, count) if count == 0: print "\tAnalyzing tables" for tab in table_list: cnt2 = session.query(tab).count() print "\t %s: %d entries" % (tab.__tablename__, cnt2) assert cnt2 == 0 print "\tRemoving migration info" mi = session.query(MigrationData).filter_by(name=unicode(mt)).one() session.delete(mi) session.commit() session.close() print "\tDropping tables" tables = [model.__table__ for model in table_list] Base_v0.metadata.drop_all(sql_db.engine, tables=tables) session.close()
def convert_media_entries(mk_db): session = Session() for entry in mk_db.MediaEntry.find().sort('created'): print repr(entry.title) new_entry = MediaEntry() copy_attrs(entry, new_entry, ('title', 'slug', 'created', 'description', 'media_type', 'state', 'license', 'fail_error', 'fail_metadata', 'queued_task_id',)) copy_reference_attr(entry, new_entry, "uploader") session.add(new_entry) session.flush() add_obj_ids(entry, new_entry) for key, value in entry.media_files.iteritems(): new_file = MediaFile(name=key, file_path=value) new_file.media_entry = new_entry.id Session.add(new_file) for attachment in entry.attachment_files: new_attach = MediaAttachmentFile( name=attachment["name"], filepath=attachment["filepath"], created=attachment["created"] ) new_attach.media_entry = new_entry.id Session.add(new_attach) session.commit() session.close()
def commit(self): Session.commit()
def atomic_update(table, query_dict, update_values): table.find(query_dict).update(update_values, synchronize_session=False) Session.commit()
def save(self, obj): Session.add(obj) Session.flush()
def setup_connection_and_db_from_config(app_config): engine = create_engine(app_config['sql_engine']) # logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) Session.configure(bind=engine) return "dummy conn", DatabaseMaster(engine)