def clean_orphan_tags(commit=True):
"""Search for unused MediaTags and delete them"""
q1 = Session.query(Tag).outerjoin(MediaTag).filter(MediaTag.id==None)
for t in q1:
Session.delete(t)
# The "let the db do all the work" version:
# q1 = Session.query(Tag.id).outerjoin(MediaTag).filter(MediaTag.id==None)
# q2 = Session.query(Tag).filter(Tag.id.in_(q1))
# q2.delete(synchronize_session = False)
if commit:
Session.commit()
def clean_orphan_tags():
q1 = Session.query(Tag).outerjoin(MediaTag).filter(MediaTag.id==None)
for t in q1:
Session.delete(t)
# The "let the db do all the work" version:
# q1 = Session.query(Tag.id).outerjoin(MediaTag).filter(MediaTag.id==None)
# q2 = Session.query(Tag).filter(Tag.id.in_(q1))
# q2.delete(synchronize_session = False)
Session.commit()
def get_test_app(dump_old_app=True):
suicide_if_bad_celery_environ()
# Make sure we've turned on testing
testing._activate_testing()
# Leave this imported as it sets up celery.
from mediagoblin.init.celery import from_tests
global MGOBLIN_APP
# Just return the old app if that exists and it's okay to set up
# and return
if MGOBLIN_APP and not dump_old_app:
return MGOBLIN_APP
Session.rollback()
Session.remove()
# Remove and reinstall user_dev directories
if os.path.exists(TEST_USER_DEV):
shutil.rmtree(TEST_USER_DEV)
for directory in USER_DEV_DIRECTORIES_TO_SETUP:
full_dir = os.path.join(TEST_USER_DEV, directory)
os.makedirs(full_dir)
# Get app config
global_config, validation_result = read_mediagoblin_config(TEST_APP_CONFIG)
app_config = global_config['mediagoblin']
# Run database setup/migrations
run_dbupdate(app_config, global_config)
# setup app and return
test_app = loadapp(
'config:' + TEST_SERVER_CONFIG)
# Re-setup celery
setup_celery_app(app_config, global_config)
# Insert the TestingMeddleware, which can do some
# sanity checks on every request/response.
# Doing it this way is probably not the cleanest way.
# We'll fix it, when we have plugins!
mg_globals.app.meddleware.insert(0, TestingMeddleware(mg_globals.app))
app = TestApp(test_app)
MGOBLIN_APP = app
return app
def check_media_slug_used(dummy_db, uploader_id, slug, ignore_m_id):
filt = (MediaEntry.uploader == uploader_id) \
& (MediaEntry.slug == slug)
if ignore_m_id is not None:
filt = filt & (MediaEntry.id != ignore_m_id)
does_exist = Session.query(MediaEntry.id).filter(filt).first() is not None
return does_exist
def check_collection_slug_used(dummy_db, creator_id, slug, ignore_c_id):
filt = (Collection.creator == creator_id) \
& (Collection.slug == slug)
if ignore_c_id is not None:
filt = filt & (Collection.id != ignore_c_id)
does_exist = Session.query(Collection.id).filter(filt).first() is not None
return does_exist
def run_conversion(config_name):
global_config, app_config = setup_global_and_app_config(config_name)
sql_conn, sql_db = sql_connect(app_config)
mk_conn, mk_db = mongo_connect(app_config)
Base_v0.metadata.create_all(sql_db.engine)
for title, func in convert_call_list:
print_header(title)
func(mk_db)
Session.remove()
for title, func in sql_call_list:
print_header(title)
func(sql_db)
Session.remove()
def media_data_init(self, **kwargs):
"""
Initialize or update the contents of a media entry's media_data row
"""
session = Session()
media_data = session.query(self.media_data_table).filter_by(
media_entry=self.id).first()
# No media data, so actually add a new one
if media_data is None:
media_data = self.media_data_table(
media_entry=self.id,
**kwargs)
session.add(media_data)
# Update old media data
else:
for field, value in kwargs.iteritems():
setattr(media_data, field, value)
def fixture_add_user(username=u'chris', password='******',
active_user=True):
test_user = mg_globals.database.User()
test_user.username = username
test_user.email = username + u'@example.com'
if password is not None:
test_user.pw_hash = bcrypt_gen_password_hash(password)
if active_user:
test_user.email_verified = True
test_user.status = u'active'
test_user.save()
# Reload
test_user = mg_globals.database.User.find_one({'username': username})
# ... and detach from session:
from mediagoblin.db.sql.base import Session
Session.expunge(test_user)
return test_user
def convert_add_migration_versions(dummy_sql_db):
session = Session()
for name in chain(("__main__",),
imap(lambda e: e[0], media_types_tables)):
print "\tAdding %s" % (name,)
m = MigrationData(name=unicode(name), version=0)
session.add(m)
session.commit()
session.close()
def convert_video(mk_db):
session = Session()
for media in mk_db.MediaEntry.find(
{'media_type': 'mediagoblin.media_types.video'}).sort('created'):
media_data_row = VideoData(**media.media_data)
media_data_row.media_entry = obj_id_table[media['_id']]
session.add(media_data_row)
session.commit()
session.close()
def convert_users(mk_db):
session = Session()
for entry in mk_db.User.find().sort('created'):
print entry.username
new_entry = User()
copy_attrs(entry, new_entry,
('username', 'email', 'created', 'pw_hash', 'email_verified',
'status', 'verification_key', 'is_admin', 'url',
'bio',
'fp_verification_key', 'fp_token_expire',))
# new_entry.fp_verification_expire = entry.fp_token_expire
session.add(new_entry)
session.flush()
add_obj_ids(entry, new_entry)
session.commit()
session.close()
def convert_image(mk_db):
session = Session()
for media in mk_db.MediaEntry.find(
{'media_type': 'mediagoblin.media_types.image'}).sort('created'):
media_data = copy(media.media_data)
if len(media_data):
media_data_row = ImageData(**media_data)
media_data_row.media_entry = obj_id_table[media['_id']]
session.add(media_data_row)
session.commit()
session.close()
def convert_media_comments(mk_db):
session = Session()
for entry in mk_db.MediaComment.find().sort('created'):
print repr(entry.content)
new_entry = MediaComment()
copy_attrs(entry, new_entry,
('created',
'content',))
try:
copy_reference_attr(entry, new_entry, "media_entry")
copy_reference_attr(entry, new_entry, "author")
except KeyError as e:
print('KeyError in convert_media_comments(): {0}'.format(e))
else:
session.add(new_entry)
session.flush()
add_obj_ids(entry, new_entry)
session.commit()
session.close()
def reset_after_request(self):
Session.rollback()
Session.remove()
def media_data(self):
session = Session()
return session.query(self.media_data_table).filter_by(
media_entry=self.id).first()
def convert_media_tags(mk_db):
session = Session()
session.autoflush = False
for media in mk_db.MediaEntry.find().sort('created'):
print repr(media.title)
for otag in media.tags:
print " ", repr((otag["slug"], otag["name"]))
nslug = session.query(Tag).filter_by(slug=otag["slug"]).first()
print " ", repr(nslug)
if nslug is None:
nslug = Tag(slug=otag["slug"])
session.add(nslug)
session.flush()
print " ", repr(nslug), nslug.id
ntag = MediaTag()
ntag.tag = nslug.id
ntag.name = otag["name"]
ntag.media_entry = obj_id_table[media._id]
session.add(ntag)
session.commit()
session.close()
def cleanup_sql_tables(sql_db):
for mt, table_list in media_types_tables:
session = Session()
count = session.query(MediaEntry.media_type). \
filter_by(media_type=unicode(mt)).count()
print " %s: %d entries" % (mt, count)
if count == 0:
print "\tAnalyzing tables"
for tab in table_list:
cnt2 = session.query(tab).count()
print "\t %s: %d entries" % (tab.__tablename__, cnt2)
assert cnt2 == 0
print "\tRemoving migration info"
mi = session.query(MigrationData).filter_by(name=unicode(mt)).one()
session.delete(mi)
session.commit()
session.close()
print "\tDropping tables"
tables = [model.__table__ for model in table_list]
Base_v0.metadata.drop_all(sql_db.engine, tables=tables)
session.close()
def convert_media_entries(mk_db):
session = Session()
for entry in mk_db.MediaEntry.find().sort('created'):
print repr(entry.title)
new_entry = MediaEntry()
copy_attrs(entry, new_entry,
('title', 'slug', 'created',
'description',
'media_type', 'state', 'license',
'fail_error', 'fail_metadata',
'queued_task_id',))
copy_reference_attr(entry, new_entry, "uploader")
session.add(new_entry)
session.flush()
add_obj_ids(entry, new_entry)
for key, value in entry.media_files.iteritems():
new_file = MediaFile(name=key, file_path=value)
new_file.media_entry = new_entry.id
Session.add(new_file)
for attachment in entry.attachment_files:
new_attach = MediaAttachmentFile(
name=attachment["name"],
filepath=attachment["filepath"],
created=attachment["created"]
)
new_attach.media_entry = new_entry.id
Session.add(new_attach)
session.commit()
session.close()
def commit(self):
Session.commit()
def atomic_update(table, query_dict, update_values):
table.find(query_dict).update(update_values,
synchronize_session=False)
Session.commit()
def save(self, obj):
Session.add(obj)
Session.flush()
def setup_connection_and_db_from_config(app_config):
engine = create_engine(app_config['sql_engine'])
# logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
Session.configure(bind=engine)
return "dummy conn", DatabaseMaster(engine)
