def adduser(args):
#TODO: Lets trust admins this do not validate Emails :)
commands_util.setup_app(args)
args.username = commands_util.prompt_if_not_set(args.username, "Username:"******"Password:"******"Email:")
db = mg_globals.database
users_with_username = \
db.User.query.filter_by(
username=args.username.lower()
).count()
if users_with_username:
print u'Sorry, a user with that name already exists.'
else:
# Create the user
entry = db.User()
entry.username = unicode(args.username.lower())
entry.email = unicode(args.email)
entry.pw_hash = auth.gen_password_hash(args.password)
entry.status = u'active'
entry.email_verified = True
entry.save()
print "User created (and email marked as verified)"
def adduser(args):
# TODO: Lets trust admins this do not validate Emails :)
commands_util.setup_app(args)
args.username = six.text_type(commands_util.prompt_if_not_set(args.username, "Username:"******"Password:"******"Email:")
db = mg_globals.database
users_with_username = db.LocalUser.query.filter(LocalUser.username == args.username.lower()).count()
if users_with_username:
print(u"Sorry, a user with that name already exists.")
sys.exit(1)
else:
# Create the user
entry = db.LocalUser()
entry.username = six.text_type(args.username.lower())
entry.email = six.text_type(args.email)
entry.pw_hash = auth.gen_password_hash(args.password)
default_privileges = [
db.Privilege.query.filter(db.Privilege.privilege_name == u"commenter").one(),
db.Privilege.query.filter(db.Privilege.privilege_name == u"uploader").one(),
db.Privilege.query.filter(db.Privilege.privilege_name == u"reporter").one(),
db.Privilege.query.filter(db.Privilege.privilege_name == u"active").one(),
]
entry.all_privileges = default_privileges
entry.save()
print(u"User created (and email marked as verified).")
def adduser(args):
#TODO: Lets trust admins this do not validate Emails :)
commands_util.setup_app(args)
args.username = commands_util.prompt_if_not_set(args.username, "Username:"******"Password:"******"Email:")
db = mg_globals.database
users_with_username = \
db.User.find({
'username': args.username.lower(),
}).count()
if users_with_username:
print u'Sorry, a user with that name already exists.'
else:
# Create the user
entry = db.User()
entry.username = unicode(args.username.lower())
entry.email = unicode(args.email)
entry.pw_hash = auth_lib.bcrypt_gen_password_hash(args.password)
entry.status = u'active'
entry.email_verified = True
entry.save(validate=True)
print "User created (and email marked as verified)"
def deleteuser(args):
commands_util.setup_app(args)
db = mg_globals.database
user = db.User.query.filter_by(username=args.username.lower()).first()
if user:
user.delete()
print('The user %s has been deleted' % args.username)
else:
print('The user %s doesn\'t exist' % args.username)
def changepw(args):
commands_util.setup_app(args)
db = mg_globals.database
user = db.User.one({'username':unicode(args.username.lower())})
if user:
user['pw_hash'] = auth_lib.bcrypt_gen_password_hash(args.password)
user.save()
print 'Password successfully changed'
else:
print 'The user doesn\'t exist'
def deleteuser(args):
commands_util.setup_app(args)
db = mg_globals.database
user = db.LocalUser.query.filter(LocalUser.username == args.username.lower()).first()
if user:
user.delete()
print("The user %s has been deleted." % args.username)
else:
print("The user %s doesn't exist." % args.username)
sys.exit(1)
def makeadmin(args):
commands_util.setup_app(args)
db = mg_globals.database
user = db.User.one({'username':unicode(args.username.lower())})
if user:
user['is_admin'] = True
user.save()
print 'The user is now Admin'
else:
print 'The user doesn\'t exist'
def changepw(args):
commands_util.setup_app(args)
db = mg_globals.database
user = db.User.query.filter_by(
username=unicode(args.username.lower())).one()
if user:
user.pw_hash = auth.gen_password_hash(args.password)
user.save()
print 'Password successfully changed'
else:
print 'The user doesn\'t exist'
def changepw(args):
commands_util.setup_app(args)
db = mg_globals.database
user = db.User.query.filter_by(
username=unicode(args.username.lower())).one()
if user:
user.pw_hash = auth.gen_password_hash(args.password)
user.save()
print 'Password successfully changed'
else:
print 'The user doesn\'t exist'
def changepw(args):
commands_util.setup_app(args)
db = mg_globals.database
user = db.LocalUser.query.filter(LocalUser.username == args.username.lower()).first()
if user:
user.pw_hash = auth.gen_password_hash(args.password)
user.save()
print(u"Password successfully changed for user %s." % args.username)
else:
print(u"The user %s doesn't exist." % args.username)
sys.exit(1)
def makeadmin(args):
commands_util.setup_app(args)
db = mg_globals.database
user = db.LocalUser.query.filter(LocalUser.username == args.username.lower()).first()
if user:
user.all_privileges.append(db.Privilege.query.filter(db.Privilege.privilege_name == u"admin").one())
user.save()
print(u"The user %s is now an admin." % args.username)
else:
print(u"The user %s doesn't exist." % args.username)
sys.exit(1)
def makeadmin(args):
commands_util.setup_app(args)
db = mg_globals.database
user = db.User.query.filter_by(
username=unicode(args.username.lower())).one()
if user:
user.is_admin = True
user.save()
print 'The user is now Admin'
else:
print 'The user doesn\'t exist'
def changepw(args):
commands_util.setup_app(args)
db = mg_globals.database
user = db.LocalUser.query.filter(
LocalUser.username == args.username.lower()).first()
if user:
user.pw_hash = auth.gen_password_hash(args.password)
user.save()
print(u'Password successfully changed for user %s.' % args.username)
else:
print(u'The user %s doesn\'t exist.' % args.username)
sys.exit(1)
def makeadmin(args):
commands_util.setup_app(args)
db = mg_globals.database
user = db.User.query.filter_by(
username=unicode(args.username.lower())).one()
if user:
user.all_privileges.append(
db.Privilege.query.filter(
db.Privilege.privilege_name == u'admin').one())
user.save()
print 'The user is now Admin'
else:
print 'The user doesn\'t exist'
def migrate(args):
mgoblin_app = commands_util.setup_app(args)
# Clear old indexes
print "== Clearing old indexes... =="
removed_indexes = db_util.remove_deprecated_indexes(mgoblin_app.db)
for collection, index_name in removed_indexes:
print "Removed index '%s' in collection '%s'" % (
index_name, collection)
# Migrate
print "== Applying migrations... =="
for model_name in migrations.MIGRATE_CLASSES:
model = getattr(mgoblin_app.db, model_name)
if not hasattr(model, 'migration_handler') or not model.collection:
continue
migration = model.migration_handler(model)
migration.migrate_all(collection=model.collection)
# Add new indexes
print "== Adding new indexes... =="
new_indexes = db_util.add_new_indexes(mgoblin_app.db)
for collection, index_name in new_indexes:
print "Added index '%s' to collection '%s'" % (
index_name, collection)
def migrate(args):
mgoblin_app = commands_util.setup_app(args)
# Clear old indexes
print "== Clearing old indexes... =="
removed_indexes = db_util.remove_deprecated_indexes(mgoblin_app.db)
for collection, index_name in removed_indexes:
print "Removed index '%s' in collection '%s'" % (index_name,
collection)
# Migrate
print "== Applying migrations... =="
for model_name in migrations.MIGRATE_CLASSES:
model = getattr(mgoblin_app.db, model_name)
if not hasattr(model, 'migration_handler') or not model.collection:
continue
migration = model.migration_handler(model)
migration.migrate_all(collection=model.collection)
# Add new indexes
print "== Adding new indexes... =="
new_indexes = db_util.add_new_indexes(mgoblin_app.db)
for collection, index_name in new_indexes:
print "Added index '%s' to collection '%s'" % (index_name, collection)
def makeadmin(args):
commands_util.setup_app(args)
db = mg_globals.database
user = db.User.query.filter_by(
username=unicode(args.username.lower())).one()
if user:
user.all_privileges.append(
db.Privilege.query.filter(
db.Privilege.privilege_name==u'admin').one()
)
user.save()
print 'The user is now Admin'
else:
print 'The user doesn\'t exist'
def makeadmin(args):
commands_util.setup_app(args)
db = mg_globals.database
user = db.LocalUser.query.filter(
LocalUser.username == args.username.lower()).first()
if user:
user.all_privileges.append(
db.Privilege.query.filter(
db.Privilege.privilege_name == u'admin').one())
user.save()
print(u'The user %s is now an admin.' % args.username)
else:
print(u'The user %s doesn\'t exist.' % args.username)
sys.exit(1)
def shell(args):
"""
Setup a shell for the user
"""
mgoblin_app = commands_util.setup_app(args)
code.interact(
banner=SHELL_BANNER,
local={
'mgoblin_app': mgoblin_app,
'mg_globals': mg_globals,
'db': mg_globals.database})
def shell(args):
"""
Setup a shell for the user
either a normal Python shell
or an IPython one
"""
user_namespace = {"mg_globals": mg_globals, "mgoblin_app": commands_util.setup_app(args), "db": mg_globals.database}
if args.ipython:
ipython_shell(**user_namespace)
else:
py_shell(**user_namespace)
def shell(args):
"""
Setup a shell for the user
"""
mgoblin_app = commands_util.setup_app(args)
code.interact(banner=SHELL_BANNER,
local={
'mgoblin_app': mgoblin_app,
'mg_globals': mg_globals,
'db': mg_globals.database
})
def reprocess(args):
# Run eagerly unless explicetly set not to
if not args.celery:
os.environ['CELERY_ALWAYS_EAGER'] = 'true'
commands_util.setup_app(args)
if args.reprocess_subcommand == "run":
run(args)
elif args.reprocess_subcommand == "available":
available(args)
elif args.reprocess_subcommand == "bulk_run":
bulk_run(args)
elif args.reprocess_subcommand == "thumbs":
thumbs(args)
elif args.reprocess_subcommand == "initial":
initial(args)
def reprocess(args):
# Run eagerly unless explicetly set not to
if not args.celery:
os.environ['CELERY_ALWAYS_EAGER'] = 'true'
commands_util.setup_app(args)
if args.reprocess_subcommand == "run":
run(args)
elif args.reprocess_subcommand == "available":
available(args)
elif args.reprocess_subcommand == "bulk_run":
bulk_run(args)
elif args.reprocess_subcommand == "thumbs":
thumbs(args)
elif args.reprocess_subcommand == "initial":
initial(args)
def adduser(args):
#TODO: Lets trust admins this do not validate Emails :)
commands_util.setup_app(args)
args.username = six.text_type(
commands_util.prompt_if_not_set(args.username, "Username:"******"Password:"******"Email:")
db = mg_globals.database
users_with_username = \
db.LocalUser.query.filter(
LocalUser.username==args.username.lower()
).count()
if users_with_username:
print(u'Sorry, a user with that name already exists.')
sys.exit(1)
else:
# Create the user
entry = db.LocalUser()
entry.username = six.text_type(args.username.lower())
entry.email = six.text_type(args.email)
entry.pw_hash = auth.gen_password_hash(args.password)
default_privileges = [
db.Privilege.query.filter(
db.Privilege.privilege_name == u'commenter').one(),
db.Privilege.query.filter(
db.Privilege.privilege_name == u'uploader').one(),
db.Privilege.query.filter(
db.Privilege.privilege_name == u'reporter').one(),
db.Privilege.query.filter(
db.Privilege.privilege_name == u'active').one()
]
entry.all_privileges = default_privileges
entry.save()
print(u"User created (and email marked as verified).")
def deletemedia(args):
app = commands_util.setup_app(args)
media_ids = set(map(int, args.media_ids.split(',')))
found_medias = set()
filter_ids = app.db.MediaEntry.id.in_(media_ids)
medias = app.db.MediaEntry.query.filter(filter_ids).all()
for media in medias:
found_medias.add(media.id)
media.delete()
print 'Media ID %d has been deleted.' % media.id
for media in media_ids - found_medias:
print 'Can\'t find a media with ID %d.' % media
print 'Done.'
def adduser(args):
#TODO: Lets trust admins this do not validate Emails :)
commands_util.setup_app(args)
args.username = commands_util.prompt_if_not_set(args.username, "Username:"******"Password:"******"Email:")
db = mg_globals.database
users_with_username = \
db.User.query.filter_by(
username=args.username.lower()
).count()
if users_with_username:
print u'Sorry, a user with that name already exists.'
else:
# Create the user
entry = db.User()
entry.username = unicode(args.username.lower())
entry.email = unicode(args.email)
entry.pw_hash = auth.gen_password_hash(args.password)
default_privileges = [
db.Privilege.query.filter(
db.Privilege.privilege_name==u'commenter').one(),
db.Privilege.query.filter(
db.Privilege.privilege_name==u'uploader').one(),
db.Privilege.query.filter(
db.Privilege.privilege_name==u'reporter').one(),
db.Privilege.query.filter(
db.Privilege.privilege_name==u'active').one()
]
entry.all_privileges = default_privileges
entry.save()
print "User created (and email marked as verified)"
def adduser(args):
#TODO: Lets trust admins this do not validate Emails :)
commands_util.setup_app(args)
db = mg_globals.database
users_with_username = \
db.User.find({
'username': args.username.lower()
}).count()
if users_with_username:
print u'Sorry, a user with that name already exists.'
else:
# Create the user
entry = db.User()
entry['username'] = unicode(args.username.lower())
entry['email'] = unicode(args.email)
entry['pw_hash'] = auth_lib.bcrypt_gen_password_hash(args.password)
entry['status'] = u'active'
entry['email_verified'] = True
entry.save(validate=True)
print "User created (and email marked as verified)"
def shell(args):
"""
Setup a shell for the user either a normal Python shell or an IPython one
"""
user_namespace = {
'mg_globals': mg_globals,
'mgoblin_app': commands_util.setup_app(args),
'db': mg_globals.database}
if args.ipython:
ipython_shell(**user_namespace)
else:
# Try ipython_shell first and fall back if not available
if not ipython_shell(**user_namespace):
py_shell(**user_namespace)
def assetlink(args):
"""
Link the asset directory of the currently installed theme and plugins
"""
mgoblin_app = commands_util.setup_app(args)
app_config = mg_globals.app_config
# link theme
link_theme_assets(mgoblin_app.current_theme, app_config['theme_linked_assets_dir'])
# link plugin assets
## ... probably for this we need the whole application initialized
for plugin_static in pluginapi.hook_runall("static_setup"):
link_plugin_assets(
plugin_static, app_config['plugin_linked_assets_dir'])
def addmedia(args):
# Run eagerly unless explicetly set not to
if not args.celery:
os.environ['CELERY_ALWAYS_EAGER'] = 'true'
app = commands_util.setup_app(args)
# get the user
user = app.db.LocalUser.query.filter(
LocalUser.username==args.username.lower()
).first()
if user is None:
print("Sorry, no user by username '%s'" % args.username)
return
# check for the file, if it exists...
filename = os.path.split(args.filename)[-1]
abs_filename = os.path.abspath(args.filename)
if not os.path.exists(abs_filename):
print("Can't find a file with filename '%s'" % args.filename)
return
upload_limit, max_file_size = get_upload_file_limits(user)
def maybe_unicodeify(some_string):
# this is kinda terrible
if some_string is None:
return None
if six.PY2:
return six.text_type(some_string, 'utf-8')
return some_string
try:
submit_media(
mg_app=app,
user=user,
submitted_file=open(abs_filename, 'rb'), filename=filename,
title=maybe_unicodeify(args.title),
description=maybe_unicodeify(args.description),
license=maybe_unicodeify(args.license),
tags_string=maybe_unicodeify(args.tags) or u"",
upload_limit=upload_limit, max_file_size=max_file_size)
except FileUploadLimit:
print("This file is larger than the upload limits for this site.")
except UserUploadLimit:
print("This file will put this user past their upload limits.")
except UserPastUploadLimit:
print("This user is already past their upload limits.")
def addmedia(args):
# Run eagerly unless explicetly set not to
if not args.celery:
os.environ['CELERY_ALWAYS_EAGER'] = 'true'
app = commands_util.setup_app(args)
# get the user
user = app.db.LocalUser.query.filter(
LocalUser.username == args.username.lower()).first()
if user is None:
print("Sorry, no user by username '%s'" % args.username)
return
# check for the file, if it exists...
filename = os.path.split(args.filename)[-1]
abs_filename = os.path.abspath(args.filename)
if not os.path.exists(abs_filename):
print("Can't find a file with filename '%s'" % args.filename)
return
upload_limit, max_file_size = get_upload_file_limits(user)
def maybe_unicodeify(some_string):
# this is kinda terrible
if some_string is None:
return None
if six.PY2:
return six.text_type(some_string, 'utf-8')
return some_string
try:
submit_media(mg_app=app,
user=user,
submitted_file=open(abs_filename, 'rb'),
filename=filename,
title=maybe_unicodeify(args.title),
description=maybe_unicodeify(args.description),
license=maybe_unicodeify(args.license),
tags_string=maybe_unicodeify(args.tags) or u"",
upload_limit=upload_limit,
max_file_size=max_file_size)
except FileUploadLimit:
print("This file is larger than the upload limits for this site.")
except UserUploadLimit:
print("This file will put this user past their upload limits.")
except UserPastUploadLimit:
print("This user is already past their upload limits.")
def assetlink(args):
"""
Link the asset directory of the currently installed theme and plugins
"""
mgoblin_app = commands_util.setup_app(args)
app_config = mg_globals.app_config
# link theme
link_theme_assets(mgoblin_app.current_theme,
app_config['theme_linked_assets_dir'])
# link plugin assets
## ... probably for this we need the whole application initialized
for plugin_static in pluginapi.hook_runall("static_setup"):
link_plugin_assets(plugin_static,
app_config['plugin_linked_assets_dir'])
def deletemedia(args):
app = commands_util.setup_app(args)
media_ids = set([int(mid) for mid in args.media_ids.split(',') if mid.isdigit()])
if not media_ids:
print 'Can\'t find any valid media ID(s).'
sys.exit(1)
found_medias = set()
filter_ids = app.db.MediaEntry.id.in_(media_ids)
medias = app.db.MediaEntry.query.filter(filter_ids).all()
for media in medias:
found_medias.add(media.id)
media.delete()
print 'Media ID %d has been deleted.' % media.id
for media in media_ids - found_medias:
print 'Can\'t find a media with ID %d.' % media
print 'Done.'
sys.exit(0)
def shell(args):
"""
Setup a shell for the user either a normal Python shell or an IPython one
"""
app = commands_util.setup_app(args)
def run_shell(db, ctx):
user_namespace = {
'mg_globals': mg_globals,
'app': app,
'db': db,
"ctx": ctx}
if args.ipython:
ipython_shell(**user_namespace)
else:
# Try ipython_shell first and fall back if not available
if not ipython_shell(**user_namespace):
py_shell(**user_namespace)
with app.gen_context() as ctx:
db = ctx.db
run_shell(db, ctx)
def batchaddmedia(args):
# Run eagerly unless explicetly set not to
if not args.celery:
os.environ['CELERY_ALWAYS_EAGER'] = 'true'
app = commands_util.setup_app(args)
files_uploaded, files_attempted = 0, 0
# get the user
user = app.db.LocalUser.query.filter(
LocalUser.username == args.username.lower()).first()
if user is None:
print(
_(u"Sorry, no user by username '{username}' exists".format(
username=args.username)))
return
temp_files = []
if os.path.isfile(args.metadata_path):
metadata_path = args.metadata_path
else:
error = _(u'File at {path} not found, use -h flag for help'.format(
path=args.metadata_path))
print(error)
return
abs_metadata_filename = os.path.abspath(metadata_path)
abs_metadata_dir = os.path.dirname(abs_metadata_filename)
def maybe_unicodeify(some_string):
# this is kinda terrible
if some_string is None:
return None
else:
return six.text_type(some_string)
with codecs.open(abs_metadata_filename, 'r',
encoding='utf-8') as all_metadata:
contents = all_metadata.read()
media_metadata = parse_csv_file(contents)
for media_id, file_metadata in media_metadata.iteritems():
files_attempted += 1
# In case the metadata was not uploaded initialize an empty dictionary.
json_ld_metadata = compact_and_validate({})
# Get all metadata entries starting with 'media' as variables and then
# delete them because those are for internal use only.
original_location = file_metadata['location']
### Pull the important media information for mediagoblin from the
### metadata, if it is provided.
title = file_metadata.get('title') or file_metadata.get('dc:title')
description = (file_metadata.get('description')
or file_metadata.get('dc:description'))
license = file_metadata.get('license')
try:
json_ld_metadata = compact_and_validate(file_metadata)
except ValidationError as exc:
error = _(
u"""Error with media '{media_id}' value '{error_path}': {error_msg}
Metadata was not uploaded.""".format(media_id=media_id,
error_path=exc.path[0],
error_msg=exc.message))
print(error)
continue
url = urlparse(original_location)
filename = url.path.split()[-1]
if url.scheme == 'http':
res = requests.get(url.geturl(), stream=True)
media_file = res.raw
elif url.scheme == '':
path = url.path
if os.path.isabs(path):
file_abs_path = os.path.abspath(path)
else:
file_path = os.path.join(abs_metadata_dir, path)
file_abs_path = os.path.abspath(file_path)
try:
media_file = file(file_abs_path, 'r')
except IOError:
print(
_(u"""\
FAIL: Local file {filename} could not be accessed.
{filename} will not be uploaded.""".format(filename=filename)))
continue
try:
submit_media(mg_app=app,
user=user,
submitted_file=media_file,
filename=filename,
title=maybe_unicodeify(title),
description=maybe_unicodeify(description),
license=maybe_unicodeify(license),
metadata=json_ld_metadata,
tags_string=u"")
print(
_(u"""Successfully submitted {filename}!
Be sure to look at the Media Processing Panel on your website to be sure it
uploaded successfully.""".format(filename=filename)))
files_uploaded += 1
except FileUploadLimit:
print(
_(u"FAIL: This file is larger than the upload limits for this site."
))
except UserUploadLimit:
print(
_("FAIL: This file will put this user past their upload limits."
))
except UserPastUploadLimit:
print(_("FAIL: This user is already past their upload limits."))
print(
_("{files_uploaded} out of {files_attempted} files successfully submitted"
.format(files_uploaded=files_uploaded,
files_attempted=files_attempted)))
def batchaddmedia(args):
# Run eagerly unless explicetly set not to
if not args.celery:
os.environ['CELERY_ALWAYS_EAGER'] = 'true'
app = commands_util.setup_app(args)
files_uploaded, files_attempted = 0, 0
# get the user
user = app.db.User.query.filter_by(username=args.username.lower()).first()
if user is None:
print _(u"Sorry, no user by username '{username}' exists".format(
username=args.username))
return
upload_limit, max_file_size = get_upload_file_limits(user)
temp_files = []
if os.path.isfile(args.metadata_path):
metadata_path = args.metadata_path
else:
error = _(u'File at {path} not found, use -h flag for help'.format(
path=args.metadata_path))
print error
return
abs_metadata_filename = os.path.abspath(metadata_path)
abs_metadata_dir = os.path.dirname(abs_metadata_filename)
upload_limit, max_file_size = get_upload_file_limits(user)
def maybe_unicodeify(some_string):
# this is kinda terrible
if some_string is None:
return None
else:
return unicode(some_string)
with codecs.open(
abs_metadata_filename, 'r', encoding='utf-8') as all_metadata:
contents = all_metadata.read()
media_metadata = parse_csv_file(contents)
for media_id, file_metadata in media_metadata.iteritems():
files_attempted += 1
# In case the metadata was not uploaded initialize an empty dictionary.
json_ld_metadata = compact_and_validate({})
# Get all metadata entries starting with 'media' as variables and then
# delete them because those are for internal use only.
original_location = file_metadata['location']
### Pull the important media information for mediagoblin from the
### metadata, if it is provided.
title = file_metadata.get('title') or file_metadata.get('dc:title')
description = (file_metadata.get('description') or
file_metadata.get('dc:description'))
license = file_metadata.get('license')
try:
json_ld_metadata = compact_and_validate(file_metadata)
except ValidationError, exc:
error = _(u"""Error with media '{media_id}' value '{error_path}': {error_msg}
Metadata was not uploaded.""".format(
media_id=media_id,
error_path=exc.path[0],
error_msg=exc.message))
print error
continue
url = urlparse(original_location)
filename = url.path.split()[-1]
if url.scheme == 'http':
res = requests.get(url.geturl(), stream=True)
media_file = res.raw
elif url.scheme == '':
path = url.path
if os.path.isabs(path):
file_abs_path = os.path.abspath(path)
else:
file_path = os.path.join(abs_metadata_dir, path)
file_abs_path = os.path.abspath(file_path)
try:
media_file = file(file_abs_path, 'r')
except IOError:
print _(u"""\
FAIL: Local file {filename} could not be accessed.
{filename} will not be uploaded.""".format(filename=filename))
continue
try:
submit_media(
mg_app=app,
user=user,
submitted_file=media_file,
filename=filename,
title=maybe_unicodeify(title),
description=maybe_unicodeify(description),
license=maybe_unicodeify(license),
metadata=json_ld_metadata,
tags_string=u"",
upload_limit=upload_limit, max_file_size=max_file_size)
print _(u"""Successfully submitted {filename}!
Be sure to look at the Media Processing Panel on your website to be sure it
uploaded successfully.""".format(filename=filename))
files_uploaded += 1
except FileUploadLimit:
print _(
u"FAIL: This file is larger than the upload limits for this site.")
except UserUploadLimit:
print _(
"FAIL: This file will put this user past their upload limits.")
except UserPastUploadLimit:
print _("FAIL: This user is already past their upload limits.")
def batchaddmedia(args):
# Run eagerly unless explicetly set not to
if not args.celery:
os.environ['CELERY_ALWAYS_EAGER'] = 'true'
app = commands_util.setup_app(args)
files_uploaded, files_attempted = 0, 0
# get the user
user = app.db.LocalUser.query.filter(
LocalUser.username == args.username.lower()).first()
if user is None:
print(
_("Sorry, no user by username '{username}' exists".format(
username=args.username)))
return
if os.path.isfile(args.metadata_path):
metadata_path = args.metadata_path
else:
error = _('File at {path} not found, use -h flag for help'.format(
path=args.metadata_path))
print(error)
return
abs_metadata_filename = os.path.abspath(metadata_path)
abs_metadata_dir = os.path.dirname(abs_metadata_filename)
all_metadata = open(abs_metadata_filename, 'r')
media_metadata = csv.DictReader(all_metadata)
for index, file_metadata in enumerate(media_metadata):
if six.PY2:
file_metadata = {
k.decode('utf-8'): v.decode('utf-8')
for k, v in file_metadata.items()
}
files_attempted += 1
# In case the metadata was not uploaded initialize an empty dictionary.
json_ld_metadata = compact_and_validate({})
# Get all metadata entries starting with 'media' as variables and then
# delete them because those are for internal use only.
original_location = file_metadata['location']
### Pull the important media information for mediagoblin from the
### metadata, if it is provided.
slug = file_metadata.get('slug')
title = file_metadata.get('title') or file_metadata.get('dc:title')
description = (file_metadata.get('description')
or file_metadata.get('dc:description'))
collection_slug = file_metadata.get('collection-slug')
license = file_metadata.get('license')
try:
json_ld_metadata = compact_and_validate(file_metadata)
except ValidationError as exc:
media_id = file_metadata.get('id') or index
error = _(
"""Error with media '{media_id}' value '{error_path}': {error_msg}
Metadata was not uploaded.""".format(media_id=media_id,
error_path=exc.path[0],
error_msg=exc.message))
print(error)
continue
if slug and MediaEntry.query.filter_by(actor=user.id,
slug=slug).count():
# Avoid re-importing media from a previous batch run. Note that this
# check isn't quite robust enough, since it requires that a slug is
# specified. Probably needs to be based on "location" since this is
# the only required field.
error = '{}: {}'.format(
slug,
_('An entry with that slug already exists for this user.'))
print(error)
continue
url = urlparse(original_location)
filename = url.path.split()[-1]
if url.scheme.startswith('http'):
res = requests.get(url.geturl(), stream=True)
if res.headers.get('content-encoding'):
# The requests library's "raw" method does not deal with content
# encoding. Alternative could be to use iter_content(), and
# write chunks to the temporary file.
raise NotImplementedError(
'URL-based media with content-encoding (eg. gzip) are not currently supported.'
)
# To avoid loading the media into memory all at once, we write it to
# a file before importing. This currently requires free space up to
# twice the size of the media file. Memory use can be tested by
# running something like `ulimit -Sv 200000` before running
# `batchaddmedia` to upload a file larger than 200MB.
media_file = tempfile.TemporaryFile()
shutil.copyfileobj(res.raw, media_file)
if six.PY2:
media_file.seek(0)
elif url.scheme == '':
path = url.path
if os.path.isabs(path):
file_abs_path = os.path.abspath(path)
else:
file_path = os.path.join(abs_metadata_dir, path)
file_abs_path = os.path.abspath(file_path)
try:
media_file = open(file_abs_path, 'rb')
except IOError:
print(
_("""\
FAIL: Local file {filename} could not be accessed.
{filename} will not be uploaded.""".format(filename=filename)))
continue
try:
entry = submit_media(mg_app=app,
user=user,
submitted_file=media_file,
filename=filename,
title=title,
description=description,
collection_slug=collection_slug,
license=license,
metadata=json_ld_metadata,
tags_string="")
if slug:
# Slug is automatically set by submit_media, so overwrite it
# with the desired slug.
entry.slug = slug
entry.save()
print(
_("""Successfully submitted {filename}!
Be sure to look at the Media Processing Panel on your website to be sure it
uploaded successfully.""".format(filename=filename)))
files_uploaded += 1
except FileUploadLimit:
print(
_("FAIL: This file is larger than the upload limits for this site."
))
except UserUploadLimit:
print(
_("FAIL: This file will put this user past their upload limits."
))
except UserPastUploadLimit:
print(_("FAIL: This user is already past their upload limits."))
finally:
media_file.close()
print(
_("{files_uploaded} out of {files_attempted} files successfully submitted"
.format(files_uploaded=files_uploaded,
files_attempted=files_attempted)))
