def adduser(args): #TODO: Lets trust admins this do not validate Emails :) commands_util.setup_app(args) args.username = commands_util.prompt_if_not_set(args.username, "Username:"******"Password:"******"Email:") db = mg_globals.database users_with_username = \ db.User.query.filter_by( username=args.username.lower() ).count() if users_with_username: print u'Sorry, a user with that name already exists.' else: # Create the user entry = db.User() entry.username = unicode(args.username.lower()) entry.email = unicode(args.email) entry.pw_hash = auth.gen_password_hash(args.password) entry.status = u'active' entry.email_verified = True entry.save() print "User created (and email marked as verified)"
def adduser(args): # TODO: Lets trust admins this do not validate Emails :) commands_util.setup_app(args) args.username = six.text_type(commands_util.prompt_if_not_set(args.username, "Username:"******"Password:"******"Email:") db = mg_globals.database users_with_username = db.LocalUser.query.filter(LocalUser.username == args.username.lower()).count() if users_with_username: print(u"Sorry, a user with that name already exists.") sys.exit(1) else: # Create the user entry = db.LocalUser() entry.username = six.text_type(args.username.lower()) entry.email = six.text_type(args.email) entry.pw_hash = auth.gen_password_hash(args.password) default_privileges = [ db.Privilege.query.filter(db.Privilege.privilege_name == u"commenter").one(), db.Privilege.query.filter(db.Privilege.privilege_name == u"uploader").one(), db.Privilege.query.filter(db.Privilege.privilege_name == u"reporter").one(), db.Privilege.query.filter(db.Privilege.privilege_name == u"active").one(), ] entry.all_privileges = default_privileges entry.save() print(u"User created (and email marked as verified).")
def adduser(args): #TODO: Lets trust admins this do not validate Emails :) commands_util.setup_app(args) args.username = commands_util.prompt_if_not_set(args.username, "Username:"******"Password:"******"Email:") db = mg_globals.database users_with_username = \ db.User.find({ 'username': args.username.lower(), }).count() if users_with_username: print u'Sorry, a user with that name already exists.' else: # Create the user entry = db.User() entry.username = unicode(args.username.lower()) entry.email = unicode(args.email) entry.pw_hash = auth_lib.bcrypt_gen_password_hash(args.password) entry.status = u'active' entry.email_verified = True entry.save(validate=True) print "User created (and email marked as verified)"
def deleteuser(args): commands_util.setup_app(args) db = mg_globals.database user = db.User.query.filter_by(username=args.username.lower()).first() if user: user.delete() print('The user %s has been deleted' % args.username) else: print('The user %s doesn\'t exist' % args.username)
def changepw(args): commands_util.setup_app(args) db = mg_globals.database user = db.User.one({'username':unicode(args.username.lower())}) if user: user['pw_hash'] = auth_lib.bcrypt_gen_password_hash(args.password) user.save() print 'Password successfully changed' else: print 'The user doesn\'t exist'
def deleteuser(args): commands_util.setup_app(args) db = mg_globals.database user = db.LocalUser.query.filter(LocalUser.username == args.username.lower()).first() if user: user.delete() print("The user %s has been deleted." % args.username) else: print("The user %s doesn't exist." % args.username) sys.exit(1)
def makeadmin(args): commands_util.setup_app(args) db = mg_globals.database user = db.User.one({'username':unicode(args.username.lower())}) if user: user['is_admin'] = True user.save() print 'The user is now Admin' else: print 'The user doesn\'t exist'
def changepw(args): commands_util.setup_app(args) db = mg_globals.database user = db.User.query.filter_by( username=unicode(args.username.lower())).one() if user: user.pw_hash = auth.gen_password_hash(args.password) user.save() print 'Password successfully changed' else: print 'The user doesn\'t exist'
def changepw(args): commands_util.setup_app(args) db = mg_globals.database user = db.LocalUser.query.filter(LocalUser.username == args.username.lower()).first() if user: user.pw_hash = auth.gen_password_hash(args.password) user.save() print(u"Password successfully changed for user %s." % args.username) else: print(u"The user %s doesn't exist." % args.username) sys.exit(1)
def makeadmin(args): commands_util.setup_app(args) db = mg_globals.database user = db.LocalUser.query.filter(LocalUser.username == args.username.lower()).first() if user: user.all_privileges.append(db.Privilege.query.filter(db.Privilege.privilege_name == u"admin").one()) user.save() print(u"The user %s is now an admin." % args.username) else: print(u"The user %s doesn't exist." % args.username) sys.exit(1)
def makeadmin(args): commands_util.setup_app(args) db = mg_globals.database user = db.User.query.filter_by( username=unicode(args.username.lower())).one() if user: user.is_admin = True user.save() print 'The user is now Admin' else: print 'The user doesn\'t exist'
def changepw(args): commands_util.setup_app(args) db = mg_globals.database user = db.LocalUser.query.filter( LocalUser.username == args.username.lower()).first() if user: user.pw_hash = auth.gen_password_hash(args.password) user.save() print(u'Password successfully changed for user %s.' % args.username) else: print(u'The user %s doesn\'t exist.' % args.username) sys.exit(1)
def makeadmin(args): commands_util.setup_app(args) db = mg_globals.database user = db.User.query.filter_by( username=unicode(args.username.lower())).one() if user: user.all_privileges.append( db.Privilege.query.filter( db.Privilege.privilege_name == u'admin').one()) user.save() print 'The user is now Admin' else: print 'The user doesn\'t exist'
def migrate(args): mgoblin_app = commands_util.setup_app(args) # Clear old indexes print "== Clearing old indexes... ==" removed_indexes = db_util.remove_deprecated_indexes(mgoblin_app.db) for collection, index_name in removed_indexes: print "Removed index '%s' in collection '%s'" % ( index_name, collection) # Migrate print "== Applying migrations... ==" for model_name in migrations.MIGRATE_CLASSES: model = getattr(mgoblin_app.db, model_name) if not hasattr(model, 'migration_handler') or not model.collection: continue migration = model.migration_handler(model) migration.migrate_all(collection=model.collection) # Add new indexes print "== Adding new indexes... ==" new_indexes = db_util.add_new_indexes(mgoblin_app.db) for collection, index_name in new_indexes: print "Added index '%s' to collection '%s'" % ( index_name, collection)
def migrate(args): mgoblin_app = commands_util.setup_app(args) # Clear old indexes print "== Clearing old indexes... ==" removed_indexes = db_util.remove_deprecated_indexes(mgoblin_app.db) for collection, index_name in removed_indexes: print "Removed index '%s' in collection '%s'" % (index_name, collection) # Migrate print "== Applying migrations... ==" for model_name in migrations.MIGRATE_CLASSES: model = getattr(mgoblin_app.db, model_name) if not hasattr(model, 'migration_handler') or not model.collection: continue migration = model.migration_handler(model) migration.migrate_all(collection=model.collection) # Add new indexes print "== Adding new indexes... ==" new_indexes = db_util.add_new_indexes(mgoblin_app.db) for collection, index_name in new_indexes: print "Added index '%s' to collection '%s'" % (index_name, collection)
def makeadmin(args): commands_util.setup_app(args) db = mg_globals.database user = db.User.query.filter_by( username=unicode(args.username.lower())).one() if user: user.all_privileges.append( db.Privilege.query.filter( db.Privilege.privilege_name==u'admin').one() ) user.save() print 'The user is now Admin' else: print 'The user doesn\'t exist'
def makeadmin(args): commands_util.setup_app(args) db = mg_globals.database user = db.LocalUser.query.filter( LocalUser.username == args.username.lower()).first() if user: user.all_privileges.append( db.Privilege.query.filter( db.Privilege.privilege_name == u'admin').one()) user.save() print(u'The user %s is now an admin.' % args.username) else: print(u'The user %s doesn\'t exist.' % args.username) sys.exit(1)
def shell(args): """ Setup a shell for the user """ mgoblin_app = commands_util.setup_app(args) code.interact( banner=SHELL_BANNER, local={ 'mgoblin_app': mgoblin_app, 'mg_globals': mg_globals, 'db': mg_globals.database})
def shell(args): """ Setup a shell for the user either a normal Python shell or an IPython one """ user_namespace = {"mg_globals": mg_globals, "mgoblin_app": commands_util.setup_app(args), "db": mg_globals.database} if args.ipython: ipython_shell(**user_namespace) else: py_shell(**user_namespace)
def shell(args): """ Setup a shell for the user """ mgoblin_app = commands_util.setup_app(args) code.interact(banner=SHELL_BANNER, local={ 'mgoblin_app': mgoblin_app, 'mg_globals': mg_globals, 'db': mg_globals.database })
def reprocess(args): # Run eagerly unless explicetly set not to if not args.celery: os.environ['CELERY_ALWAYS_EAGER'] = 'true' commands_util.setup_app(args) if args.reprocess_subcommand == "run": run(args) elif args.reprocess_subcommand == "available": available(args) elif args.reprocess_subcommand == "bulk_run": bulk_run(args) elif args.reprocess_subcommand == "thumbs": thumbs(args) elif args.reprocess_subcommand == "initial": initial(args)
def adduser(args): #TODO: Lets trust admins this do not validate Emails :) commands_util.setup_app(args) args.username = six.text_type( commands_util.prompt_if_not_set(args.username, "Username:"******"Password:"******"Email:") db = mg_globals.database users_with_username = \ db.LocalUser.query.filter( LocalUser.username==args.username.lower() ).count() if users_with_username: print(u'Sorry, a user with that name already exists.') sys.exit(1) else: # Create the user entry = db.LocalUser() entry.username = six.text_type(args.username.lower()) entry.email = six.text_type(args.email) entry.pw_hash = auth.gen_password_hash(args.password) default_privileges = [ db.Privilege.query.filter( db.Privilege.privilege_name == u'commenter').one(), db.Privilege.query.filter( db.Privilege.privilege_name == u'uploader').one(), db.Privilege.query.filter( db.Privilege.privilege_name == u'reporter').one(), db.Privilege.query.filter( db.Privilege.privilege_name == u'active').one() ] entry.all_privileges = default_privileges entry.save() print(u"User created (and email marked as verified).")
def deletemedia(args): app = commands_util.setup_app(args) media_ids = set(map(int, args.media_ids.split(','))) found_medias = set() filter_ids = app.db.MediaEntry.id.in_(media_ids) medias = app.db.MediaEntry.query.filter(filter_ids).all() for media in medias: found_medias.add(media.id) media.delete() print 'Media ID %d has been deleted.' % media.id for media in media_ids - found_medias: print 'Can\'t find a media with ID %d.' % media print 'Done.'
def adduser(args): #TODO: Lets trust admins this do not validate Emails :) commands_util.setup_app(args) args.username = commands_util.prompt_if_not_set(args.username, "Username:"******"Password:"******"Email:") db = mg_globals.database users_with_username = \ db.User.query.filter_by( username=args.username.lower() ).count() if users_with_username: print u'Sorry, a user with that name already exists.' else: # Create the user entry = db.User() entry.username = unicode(args.username.lower()) entry.email = unicode(args.email) entry.pw_hash = auth.gen_password_hash(args.password) default_privileges = [ db.Privilege.query.filter( db.Privilege.privilege_name==u'commenter').one(), db.Privilege.query.filter( db.Privilege.privilege_name==u'uploader').one(), db.Privilege.query.filter( db.Privilege.privilege_name==u'reporter').one(), db.Privilege.query.filter( db.Privilege.privilege_name==u'active').one() ] entry.all_privileges = default_privileges entry.save() print "User created (and email marked as verified)"
def adduser(args): #TODO: Lets trust admins this do not validate Emails :) commands_util.setup_app(args) db = mg_globals.database users_with_username = \ db.User.find({ 'username': args.username.lower() }).count() if users_with_username: print u'Sorry, a user with that name already exists.' else: # Create the user entry = db.User() entry['username'] = unicode(args.username.lower()) entry['email'] = unicode(args.email) entry['pw_hash'] = auth_lib.bcrypt_gen_password_hash(args.password) entry['status'] = u'active' entry['email_verified'] = True entry.save(validate=True) print "User created (and email marked as verified)"
def shell(args): """ Setup a shell for the user either a normal Python shell or an IPython one """ user_namespace = { 'mg_globals': mg_globals, 'mgoblin_app': commands_util.setup_app(args), 'db': mg_globals.database} if args.ipython: ipython_shell(**user_namespace) else: # Try ipython_shell first and fall back if not available if not ipython_shell(**user_namespace): py_shell(**user_namespace)
def assetlink(args): """ Link the asset directory of the currently installed theme and plugins """ mgoblin_app = commands_util.setup_app(args) app_config = mg_globals.app_config # link theme link_theme_assets(mgoblin_app.current_theme, app_config['theme_linked_assets_dir']) # link plugin assets ## ... probably for this we need the whole application initialized for plugin_static in pluginapi.hook_runall("static_setup"): link_plugin_assets( plugin_static, app_config['plugin_linked_assets_dir'])
def addmedia(args): # Run eagerly unless explicetly set not to if not args.celery: os.environ['CELERY_ALWAYS_EAGER'] = 'true' app = commands_util.setup_app(args) # get the user user = app.db.LocalUser.query.filter( LocalUser.username==args.username.lower() ).first() if user is None: print("Sorry, no user by username '%s'" % args.username) return # check for the file, if it exists... filename = os.path.split(args.filename)[-1] abs_filename = os.path.abspath(args.filename) if not os.path.exists(abs_filename): print("Can't find a file with filename '%s'" % args.filename) return upload_limit, max_file_size = get_upload_file_limits(user) def maybe_unicodeify(some_string): # this is kinda terrible if some_string is None: return None if six.PY2: return six.text_type(some_string, 'utf-8') return some_string try: submit_media( mg_app=app, user=user, submitted_file=open(abs_filename, 'rb'), filename=filename, title=maybe_unicodeify(args.title), description=maybe_unicodeify(args.description), license=maybe_unicodeify(args.license), tags_string=maybe_unicodeify(args.tags) or u"", upload_limit=upload_limit, max_file_size=max_file_size) except FileUploadLimit: print("This file is larger than the upload limits for this site.") except UserUploadLimit: print("This file will put this user past their upload limits.") except UserPastUploadLimit: print("This user is already past their upload limits.")
def addmedia(args): # Run eagerly unless explicetly set not to if not args.celery: os.environ['CELERY_ALWAYS_EAGER'] = 'true' app = commands_util.setup_app(args) # get the user user = app.db.LocalUser.query.filter( LocalUser.username == args.username.lower()).first() if user is None: print("Sorry, no user by username '%s'" % args.username) return # check for the file, if it exists... filename = os.path.split(args.filename)[-1] abs_filename = os.path.abspath(args.filename) if not os.path.exists(abs_filename): print("Can't find a file with filename '%s'" % args.filename) return upload_limit, max_file_size = get_upload_file_limits(user) def maybe_unicodeify(some_string): # this is kinda terrible if some_string is None: return None if six.PY2: return six.text_type(some_string, 'utf-8') return some_string try: submit_media(mg_app=app, user=user, submitted_file=open(abs_filename, 'rb'), filename=filename, title=maybe_unicodeify(args.title), description=maybe_unicodeify(args.description), license=maybe_unicodeify(args.license), tags_string=maybe_unicodeify(args.tags) or u"", upload_limit=upload_limit, max_file_size=max_file_size) except FileUploadLimit: print("This file is larger than the upload limits for this site.") except UserUploadLimit: print("This file will put this user past their upload limits.") except UserPastUploadLimit: print("This user is already past their upload limits.")
def assetlink(args): """ Link the asset directory of the currently installed theme and plugins """ mgoblin_app = commands_util.setup_app(args) app_config = mg_globals.app_config # link theme link_theme_assets(mgoblin_app.current_theme, app_config['theme_linked_assets_dir']) # link plugin assets ## ... probably for this we need the whole application initialized for plugin_static in pluginapi.hook_runall("static_setup"): link_plugin_assets(plugin_static, app_config['plugin_linked_assets_dir'])
def deletemedia(args): app = commands_util.setup_app(args) media_ids = set([int(mid) for mid in args.media_ids.split(',') if mid.isdigit()]) if not media_ids: print 'Can\'t find any valid media ID(s).' sys.exit(1) found_medias = set() filter_ids = app.db.MediaEntry.id.in_(media_ids) medias = app.db.MediaEntry.query.filter(filter_ids).all() for media in medias: found_medias.add(media.id) media.delete() print 'Media ID %d has been deleted.' % media.id for media in media_ids - found_medias: print 'Can\'t find a media with ID %d.' % media print 'Done.' sys.exit(0)
def shell(args): """ Setup a shell for the user either a normal Python shell or an IPython one """ app = commands_util.setup_app(args) def run_shell(db, ctx): user_namespace = { 'mg_globals': mg_globals, 'app': app, 'db': db, "ctx": ctx} if args.ipython: ipython_shell(**user_namespace) else: # Try ipython_shell first and fall back if not available if not ipython_shell(**user_namespace): py_shell(**user_namespace) with app.gen_context() as ctx: db = ctx.db run_shell(db, ctx)
def batchaddmedia(args): # Run eagerly unless explicetly set not to if not args.celery: os.environ['CELERY_ALWAYS_EAGER'] = 'true' app = commands_util.setup_app(args) files_uploaded, files_attempted = 0, 0 # get the user user = app.db.LocalUser.query.filter( LocalUser.username == args.username.lower()).first() if user is None: print( _(u"Sorry, no user by username '{username}' exists".format( username=args.username))) return temp_files = [] if os.path.isfile(args.metadata_path): metadata_path = args.metadata_path else: error = _(u'File at {path} not found, use -h flag for help'.format( path=args.metadata_path)) print(error) return abs_metadata_filename = os.path.abspath(metadata_path) abs_metadata_dir = os.path.dirname(abs_metadata_filename) def maybe_unicodeify(some_string): # this is kinda terrible if some_string is None: return None else: return six.text_type(some_string) with codecs.open(abs_metadata_filename, 'r', encoding='utf-8') as all_metadata: contents = all_metadata.read() media_metadata = parse_csv_file(contents) for media_id, file_metadata in media_metadata.iteritems(): files_attempted += 1 # In case the metadata was not uploaded initialize an empty dictionary. json_ld_metadata = compact_and_validate({}) # Get all metadata entries starting with 'media' as variables and then # delete them because those are for internal use only. original_location = file_metadata['location'] ### Pull the important media information for mediagoblin from the ### metadata, if it is provided. title = file_metadata.get('title') or file_metadata.get('dc:title') description = (file_metadata.get('description') or file_metadata.get('dc:description')) license = file_metadata.get('license') try: json_ld_metadata = compact_and_validate(file_metadata) except ValidationError as exc: error = _( u"""Error with media '{media_id}' value '{error_path}': {error_msg} Metadata was not uploaded.""".format(media_id=media_id, error_path=exc.path[0], error_msg=exc.message)) print(error) continue url = urlparse(original_location) filename = url.path.split()[-1] if url.scheme == 'http': res = requests.get(url.geturl(), stream=True) media_file = res.raw elif url.scheme == '': path = url.path if os.path.isabs(path): file_abs_path = os.path.abspath(path) else: file_path = os.path.join(abs_metadata_dir, path) file_abs_path = os.path.abspath(file_path) try: media_file = file(file_abs_path, 'r') except IOError: print( _(u"""\ FAIL: Local file {filename} could not be accessed. {filename} will not be uploaded.""".format(filename=filename))) continue try: submit_media(mg_app=app, user=user, submitted_file=media_file, filename=filename, title=maybe_unicodeify(title), description=maybe_unicodeify(description), license=maybe_unicodeify(license), metadata=json_ld_metadata, tags_string=u"") print( _(u"""Successfully submitted {filename}! Be sure to look at the Media Processing Panel on your website to be sure it uploaded successfully.""".format(filename=filename))) files_uploaded += 1 except FileUploadLimit: print( _(u"FAIL: This file is larger than the upload limits for this site." )) except UserUploadLimit: print( _("FAIL: This file will put this user past their upload limits." )) except UserPastUploadLimit: print(_("FAIL: This user is already past their upload limits.")) print( _("{files_uploaded} out of {files_attempted} files successfully submitted" .format(files_uploaded=files_uploaded, files_attempted=files_attempted)))
def batchaddmedia(args): # Run eagerly unless explicetly set not to if not args.celery: os.environ['CELERY_ALWAYS_EAGER'] = 'true' app = commands_util.setup_app(args) files_uploaded, files_attempted = 0, 0 # get the user user = app.db.User.query.filter_by(username=args.username.lower()).first() if user is None: print _(u"Sorry, no user by username '{username}' exists".format( username=args.username)) return upload_limit, max_file_size = get_upload_file_limits(user) temp_files = [] if os.path.isfile(args.metadata_path): metadata_path = args.metadata_path else: error = _(u'File at {path} not found, use -h flag for help'.format( path=args.metadata_path)) print error return abs_metadata_filename = os.path.abspath(metadata_path) abs_metadata_dir = os.path.dirname(abs_metadata_filename) upload_limit, max_file_size = get_upload_file_limits(user) def maybe_unicodeify(some_string): # this is kinda terrible if some_string is None: return None else: return unicode(some_string) with codecs.open( abs_metadata_filename, 'r', encoding='utf-8') as all_metadata: contents = all_metadata.read() media_metadata = parse_csv_file(contents) for media_id, file_metadata in media_metadata.iteritems(): files_attempted += 1 # In case the metadata was not uploaded initialize an empty dictionary. json_ld_metadata = compact_and_validate({}) # Get all metadata entries starting with 'media' as variables and then # delete them because those are for internal use only. original_location = file_metadata['location'] ### Pull the important media information for mediagoblin from the ### metadata, if it is provided. title = file_metadata.get('title') or file_metadata.get('dc:title') description = (file_metadata.get('description') or file_metadata.get('dc:description')) license = file_metadata.get('license') try: json_ld_metadata = compact_and_validate(file_metadata) except ValidationError, exc: error = _(u"""Error with media '{media_id}' value '{error_path}': {error_msg} Metadata was not uploaded.""".format( media_id=media_id, error_path=exc.path[0], error_msg=exc.message)) print error continue url = urlparse(original_location) filename = url.path.split()[-1] if url.scheme == 'http': res = requests.get(url.geturl(), stream=True) media_file = res.raw elif url.scheme == '': path = url.path if os.path.isabs(path): file_abs_path = os.path.abspath(path) else: file_path = os.path.join(abs_metadata_dir, path) file_abs_path = os.path.abspath(file_path) try: media_file = file(file_abs_path, 'r') except IOError: print _(u"""\ FAIL: Local file {filename} could not be accessed. {filename} will not be uploaded.""".format(filename=filename)) continue try: submit_media( mg_app=app, user=user, submitted_file=media_file, filename=filename, title=maybe_unicodeify(title), description=maybe_unicodeify(description), license=maybe_unicodeify(license), metadata=json_ld_metadata, tags_string=u"", upload_limit=upload_limit, max_file_size=max_file_size) print _(u"""Successfully submitted {filename}! Be sure to look at the Media Processing Panel on your website to be sure it uploaded successfully.""".format(filename=filename)) files_uploaded += 1 except FileUploadLimit: print _( u"FAIL: This file is larger than the upload limits for this site.") except UserUploadLimit: print _( "FAIL: This file will put this user past their upload limits.") except UserPastUploadLimit: print _("FAIL: This user is already past their upload limits.")
def batchaddmedia(args): # Run eagerly unless explicetly set not to if not args.celery: os.environ['CELERY_ALWAYS_EAGER'] = 'true' app = commands_util.setup_app(args) files_uploaded, files_attempted = 0, 0 # get the user user = app.db.LocalUser.query.filter( LocalUser.username == args.username.lower()).first() if user is None: print( _("Sorry, no user by username '{username}' exists".format( username=args.username))) return if os.path.isfile(args.metadata_path): metadata_path = args.metadata_path else: error = _('File at {path} not found, use -h flag for help'.format( path=args.metadata_path)) print(error) return abs_metadata_filename = os.path.abspath(metadata_path) abs_metadata_dir = os.path.dirname(abs_metadata_filename) all_metadata = open(abs_metadata_filename, 'r') media_metadata = csv.DictReader(all_metadata) for index, file_metadata in enumerate(media_metadata): if six.PY2: file_metadata = { k.decode('utf-8'): v.decode('utf-8') for k, v in file_metadata.items() } files_attempted += 1 # In case the metadata was not uploaded initialize an empty dictionary. json_ld_metadata = compact_and_validate({}) # Get all metadata entries starting with 'media' as variables and then # delete them because those are for internal use only. original_location = file_metadata['location'] ### Pull the important media information for mediagoblin from the ### metadata, if it is provided. slug = file_metadata.get('slug') title = file_metadata.get('title') or file_metadata.get('dc:title') description = (file_metadata.get('description') or file_metadata.get('dc:description')) collection_slug = file_metadata.get('collection-slug') license = file_metadata.get('license') try: json_ld_metadata = compact_and_validate(file_metadata) except ValidationError as exc: media_id = file_metadata.get('id') or index error = _( """Error with media '{media_id}' value '{error_path}': {error_msg} Metadata was not uploaded.""".format(media_id=media_id, error_path=exc.path[0], error_msg=exc.message)) print(error) continue if slug and MediaEntry.query.filter_by(actor=user.id, slug=slug).count(): # Avoid re-importing media from a previous batch run. Note that this # check isn't quite robust enough, since it requires that a slug is # specified. Probably needs to be based on "location" since this is # the only required field. error = '{}: {}'.format( slug, _('An entry with that slug already exists for this user.')) print(error) continue url = urlparse(original_location) filename = url.path.split()[-1] if url.scheme.startswith('http'): res = requests.get(url.geturl(), stream=True) if res.headers.get('content-encoding'): # The requests library's "raw" method does not deal with content # encoding. Alternative could be to use iter_content(), and # write chunks to the temporary file. raise NotImplementedError( 'URL-based media with content-encoding (eg. gzip) are not currently supported.' ) # To avoid loading the media into memory all at once, we write it to # a file before importing. This currently requires free space up to # twice the size of the media file. Memory use can be tested by # running something like `ulimit -Sv 200000` before running # `batchaddmedia` to upload a file larger than 200MB. media_file = tempfile.TemporaryFile() shutil.copyfileobj(res.raw, media_file) if six.PY2: media_file.seek(0) elif url.scheme == '': path = url.path if os.path.isabs(path): file_abs_path = os.path.abspath(path) else: file_path = os.path.join(abs_metadata_dir, path) file_abs_path = os.path.abspath(file_path) try: media_file = open(file_abs_path, 'rb') except IOError: print( _("""\ FAIL: Local file {filename} could not be accessed. {filename} will not be uploaded.""".format(filename=filename))) continue try: entry = submit_media(mg_app=app, user=user, submitted_file=media_file, filename=filename, title=title, description=description, collection_slug=collection_slug, license=license, metadata=json_ld_metadata, tags_string="") if slug: # Slug is automatically set by submit_media, so overwrite it # with the desired slug. entry.slug = slug entry.save() print( _("""Successfully submitted {filename}! Be sure to look at the Media Processing Panel on your website to be sure it uploaded successfully.""".format(filename=filename))) files_uploaded += 1 except FileUploadLimit: print( _("FAIL: This file is larger than the upload limits for this site." )) except UserUploadLimit: print( _("FAIL: This file will put this user past their upload limits." )) except UserPastUploadLimit: print(_("FAIL: This user is already past their upload limits.")) finally: media_file.close() print( _("{files_uploaded} out of {files_attempted} files successfully submitted" .format(files_uploaded=files_uploaded, files_attempted=files_attempted)))