class CreateUserCommand(superdesk.Command):
"""Create a user with given username, password and email.
If user with given username exists, reset password.
"""
option_list = (
superdesk.Option('--username', '-u', dest='username', required=True),
superdesk.Option('--password', '-p', dest='password', required=True),
superdesk.Option('--email', '-e', dest='email', required=True),
)
def run(self, username, password, email):
userdata = {
'username': username,
'password': password,
'email': email,
}
user = superdesk.app.data.find_one('users', username=userdata.get('username'), req=None)
if user:
userdata[app.config['LAST_UPDATED']] = utcnow()
userdata['password'] = hash_password(userdata['password'])
superdesk.apps['users'].update(user.get('_id'), userdata, trigger_events=False)
return userdata
else:
userdata[app.config['DATE_CREATED']] = utcnow()
userdata[app.config['LAST_UPDATED']] = utcnow()
superdesk.apps['users'].create([userdata], trigger_events=True)
return userdata
class RunMacro(superdesk.Command):
r"""Executes a macro by given name and optional keyword arguments.
Example:
::
$ app:run_macro --name clean_keywords --kwargs {"repo":"archived"}
"""
option_list = [
superdesk.Option('--name', '-n', dest='macro_name', required=True),
superdesk.Option('--kwargs', '-k', dest='kwargs', required=False)
]
def run(self, macro_name, kwargs):
kwargs = json.loads(kwargs)
macro = superdesk.get_resource_service('macros').get_macro_by_name(
macro_name)
if not macro:
print('Failed to locate macro {}.'.format(macro_name))
return
macro['callback'](**kwargs)
class UpdateIngest(superdesk.Command):
"""Runs update for ingest providers.
Example:
::
$ python manage.py ingest:update
$ python manage.py ingest:update --provider=aap-demo
"""
option_list = (
superdesk.Option("--provider", "-p", dest="provider_name"),
superdesk.Option("--sync", "-s", dest="sync", action="store_true"),
)
def run(self, provider_name=None, sync=False):
lookup = {} if not provider_name else {"name": provider_name}
for provider in superdesk.get_resource_service("ingest_providers").get(req=None, lookup=lookup):
if (
not is_closed(provider)
and is_service_and_parser_registered(provider)
and (is_scheduled(provider) or sync)
):
kwargs = {
"provider": provider,
"rule_set": get_provider_rule_set(provider),
"routing_scheme": get_provider_routing_scheme(provider),
"sync": sync,
}
if sync:
update_provider.apply(kwargs=kwargs)
else:
update_provider.apply_async(expires=get_task_ttl(provider), kwargs=kwargs, serializer="eve/json")
class RunMacro(superdesk.Command):
r"""Executes a macro by given name and optional keyword arguments.
Example:
::
$ app:run_macro --name clean_keywords --kwargs {"repo":"archived"}
"""
option_list = [
superdesk.Option("--name", "-n", dest="macro_name", required=True),
superdesk.Option("--kwargs", "-k", dest="kwargs", required=False),
]
def run(self, macro_name, kwargs):
kwargs = json.loads(kwargs)
macro = superdesk.get_resource_service("macros").get_macro_by_name(
macro_name)
if not macro:
print("Failed to locate macro {}.".format(macro_name))
return
macro["callback"](**kwargs)
class AppPrepopulateCommand(superdesk.Command):
"""Prepopulate Superdesk using sample data.
Useful for demo/development environment, but don't run in production,
it's hard to get rid of such data later.
Example:
::
$ python manage.py app:prepopulate
"""
option_list = [
superdesk.Option("--file",
"-f",
dest="prepopulate_file",
default="app_prepopulate_data.json"),
superdesk.Option("--dir", "-d", dest="directory", default=None),
]
def run(self, prepopulate_file, directory=None):
user = get_resource_service("users").find_one(
username=get_default_user()["username"], req=None)
if not user:
get_resource_service("users").post([get_default_user()])
prepopulate_data(prepopulate_file, get_default_user(), directory)
class GetAuthTokenCommand(superdesk.Command):
"""Gets auth token.
Generate an authorization token to be able to authenticate against the REST api without
starting the client the copy the authorization header.
Example:
::
$ python manage.py users:get_auth_token --username=admin --password=123123
"""
option_list = (
superdesk.Option("--username", "-u", dest="username", required=True),
superdesk.Option("--password", "-p", dest="password", required=True),
)
def run(self, username, password):
credentials = {"username": username, "password": password}
service = superdesk.get_resource_service("auth_db")
id = str(service.post([credentials])[0])
print("Session ID:", id)
creds = service.find_one(req=None, _id=id)
token = creds.get("token").encode("ascii")
encoded_token = b"basic " + b64encode(token + b":")
print("Generated token: ", encoded_token)
return encoded_token
class UpdateIngest(superdesk.Command):
"""Runs update for ingest providers.
Example:
::
$ python manage.py ingest:update
$ python manage.py ingest:update --provider=aap-demo
"""
option_list = (
superdesk.Option('--provider', '-p', dest='provider_name'),
superdesk.Option('--sync', '-s', dest='sync', action='store_true'),
)
def run(self, provider_name=None, sync=False):
lookup = {} if not provider_name else {'name': provider_name}
for provider in superdesk.get_resource_service('ingest_providers').get(req=None, lookup=lookup):
if not is_closed(provider) and is_service_and_parser_registered(provider) and \
(is_scheduled(provider) or sync):
kwargs = {
'provider': provider,
'rule_set': get_provider_rule_set(provider),
'routing_scheme': get_provider_routing_scheme(provider),
'sync': sync,
}
if sync:
update_provider.apply(kwargs=kwargs)
else:
update_provider.apply_async(expires=get_task_ttl(provider), kwargs=kwargs)
class GetAuthTokenCommand(superdesk.Command):
"""Gets auth token.
Generate an authorization token to be able to authenticate against the REST api without
starting the client the copy the authorization header.
"""
option_list = (
superdesk.Option('--username', '-u', dest='username', required=True),
superdesk.Option('--password', '-p', dest='password', required=True)
)
def run(self, username, password):
credentials = {
'username': username,
'password': password
}
service = superdesk.get_resource_service('auth_db')
id = str(service.post([credentials])[0])
print('Session ID:', id)
creds = service.find_one(req=None, _id=id)
token = creds.get('token').encode('ascii')
encoded_token = b'basic ' + b64encode(token + b':')
print('Generated token: ', encoded_token)
return encoded_token
class CreateUserCommand(superdesk.Command):
"""Create a user with given username, password and email.
If user with given username exists it's noop.
Example:
::
$ python manage.py users:create -u admin -p admin -e '*****@*****.**' --admin
"""
option_list = (
superdesk.Option('--username', '-u', dest='username', required=True),
superdesk.Option('--password', '-p', dest='password', required=True),
superdesk.Option('--email', '-e', dest='email', required=True),
superdesk.Option('--admin',
'-a',
dest='admin',
required=False,
action='store_true'),
superdesk.Option('--support',
'-s',
dest='support',
required=False,
action='store_true'),
)
def run(self, username, password, email, admin=False, support=False):
# force type conversion to boolean
user_type = 'administrator' if admin else 'user'
userdata = {
'username': username,
'password': password,
'email': email,
'user_type': user_type,
'is_active': admin,
'is_support': support,
'needs_activation': not admin
}
with app.test_request_context('/users', method='POST'):
if userdata.get('password',
None) and not is_hashed(userdata.get('password')):
userdata['password'] = get_hash(
userdata.get('password'),
app.config.get('BCRYPT_GENSALT_WORK_FACTOR', 12))
user = superdesk.get_resource_service('users').find_one(
username=userdata.get('username'), req=None)
if user:
logger.info('user already exists %s' % (userdata))
else:
logger.info('creating user %s' % (userdata))
superdesk.get_resource_service('users').post([userdata])
logger.info('user saved %s' % (userdata))
return userdata
class GenerateUpdate(superdesk.Command):
"""Generate a file where to define a new data update.
Example:
::
$ python manage.py data:generate_update --resource=archive
"""
option_list = [
superdesk.Option("--resource",
"-r",
dest="resource_name",
required=True,
help="Resource to update"),
superdesk.Option(
"--global",
"-g",
dest="global_update",
required=False,
action="store_true",
help="This data update belongs to superdesk core",
),
]
def run(self, resource_name, global_update=False):
timestamp = time.strftime("%Y%m%d-%H%M%S")
# create a data update file
try:
last_file = get_data_updates_files()[-1]
name_id = int(last_file.replace(".py", "").split("_")[0]) + 1
except IndexError:
name_id = 0
if global_update:
update_dir = MAIN_DATA_UPDATES_DIR
else:
update_dir = get_dirs(only_relative_folder=True)[0]
if not os.path.exists(update_dir):
os.makedirs(update_dir)
data_update_filename = os.path.join(
update_dir, "{:05d}_{}_{}.py".format(name_id, timestamp,
resource_name))
if os.path.exists(data_update_filename):
raise Exception('The file "%s" already exists' %
(data_update_filename))
with open(data_update_filename, "w+") as f:
template_context = {
"resource": resource_name,
"current_date": time.strftime("%Y-%m-%d %H:%M"),
"user": getpass.getuser(),
"default_fw_implementation":
DEFAULT_DATA_UPDATE_FW_IMPLEMENTATION,
"default_bw_implementation":
DEFAULT_DATA_UPDATE_BW_IMPLEMENTATION,
}
f.write(
Template(DATA_UPDATE_TEMPLATE).substitute(template_context))
print("Data update file created %s" % (data_update_filename))
class ImportUserProfileFromADCommand(superdesk.Command):
"""Responsible for importing a user profile from Active Directory (AD) to Mongo.
This command runs on assumption that the user executing this command and
the user whose profile need to be imported need not to be the same. Uses ad_username and ad_password to bind to AD
and then searches for a user identified by username_to_import and if found imports into Mongo.
Example:
::
$ python manage.py users:copyfromad --ad_username=ad_uname --ad_password=123 --username_to_import=admin
"""
option_list = (
superdesk.Option("--ad_username", "-adu", dest="ad_username", required=True),
superdesk.Option("--ad_password", "-adp", dest="ad_password", required=True),
superdesk.Option("--username_to_import", "-u", dest="username", required=True),
superdesk.Option("--admin", "-a", dest="admin", required=False),
)
def run(self, ad_username, ad_password, username, admin="false"):
"""Imports or Updates a User Profile from AD to Mongo.
:param ad_username: Active Directory Username
:param ad_password: Password of Active Directory Username
:param username: Username as in Active Directory whose profile needs to be imported to Superdesk.
:return: User Profile.
"""
# force type conversion to boolean
user_type = "administrator" if admin is not None and admin.lower() == "true" else "user"
# Authenticate and fetch profile from AD
settings = app.settings
ad_auth = ADAuth(
settings["LDAP_SERVER"],
settings["LDAP_SERVER_PORT"],
settings["LDAP_BASE_FILTER"],
settings["LDAP_USER_FILTER"],
settings["LDAP_USER_ATTRIBUTES"],
settings["LDAP_FQDN"],
)
user_data = ad_auth.authenticate_and_fetch_profile(ad_username, ad_password, username)
if len(user_data) == 0:
raise SuperdeskApiError.notFoundError("Username not found")
# Check if User Profile already exists in Mongo
user = superdesk.get_resource_service("users").find_one(req=None, **get_user_query(username))
if user:
superdesk.get_resource_service("users").patch(user.get("_id"), user_data)
else:
add_default_values(user_data, username, user_type=user_type)
superdesk.get_resource_service("users").post([user_data])
return user_data
class GenerateUpdate(superdesk.Command):
"""Generate a file where to define a new data update.
Example:
::
$ python manage.py data:generate_update --resource=archive
"""
option_list = [
superdesk.Option('--resource',
'-r',
dest='resource_name',
required=True,
help='Resource to update'),
superdesk.Option('--global',
'-g',
dest='global_update',
required=False,
action='store_true',
help='This data update belongs to superdesk core'),
]
def run(self, resource_name, global_update=False):
timestamp = time.strftime('%Y%m%d-%H%M%S')
# create a data update file
try:
last_file = get_data_updates_files()[-1]
name_id = int(last_file.replace('.py', '').split('_')[0]) + 1
except IndexError:
name_id = 0
if global_update:
update_dir = MAIN_DATA_UPDATES_DIR
else:
update_dir = get_dirs(only_relative_folder=True)[0]
if not os.path.exists(update_dir):
os.makedirs(update_dir)
data_update_filename = os.path.join(
update_dir, '{:05d}_{}_{}.py'.format(name_id, timestamp,
resource_name))
if os.path.exists(data_update_filename):
raise Exception('The file "%s" already exists' %
(data_update_filename))
with open(data_update_filename, 'w+') as f:
template_context = {
'resource': resource_name,
'current_date': time.strftime("%Y-%m-%d %H:%M"),
'user': getpass.getuser(),
'default_fw_implementation':
DEFAULT_DATA_UPDATE_FW_IMPLEMENTATION,
'default_bw_implementation':
DEFAULT_DATA_UPDATE_BW_IMPLEMENTATION
}
f.write(
Template(DATA_UPDATE_TEMPLATE).substitute(template_context))
print('Data update file created %s' % (data_update_filename))
class MigrateMediaCommand(superdesk.Command):
"""It will migrate files from Mongo GridFS into Amazon S3 storage."""
option_list = [
superdesk.Option("--limit",
"-l",
dest="limit",
required=False,
default=50,
type=int),
superdesk.Option("--skip",
"-s",
dest="skip",
required=False,
default=0,
type=int),
superdesk.Option("--delete",
"-d",
dest="delete",
required=False,
default=False,
action="store_true"),
]
def run(self, limit, skip, delete):
mongo = app.media._storage[1]
amazon = app.media._storage[0]
files = mongo.fs().find(no_cursor_timeout=True).limit(limit).skip(skip)
if not files.count():
print("There are no files in mongo to be migrated.")
return
print("starting to migrate {} files".format(files.count()))
migrated = 0
for file in files:
try:
saved = amazon.put(
file.read(),
filename=file.filename,
content_type=file.content_type,
metadata=file.metadata,
_id=str(file._id),
ContentMD5=codecs.encode(codecs.decode(file.md5, "hex"),
"base64").decode().strip(),
)
if saved:
if delete:
mongo.delete(file._id)
migrated += 1
print(".", end="")
except Exception as error:
print("Error while migrating file {}: {}".format(
file._id, error))
print("done migrating {} files.".format(migrated))
class CreateUserCommand(superdesk.Command):
"""Create a user with given username, password and email.
If user with given username exists it's noop.
Example:
::
$ python manage.py users:create -u admin -p admin -e '*****@*****.**' --admin
"""
option_list = (
superdesk.Option("--username", "-u", dest="username", required=True),
superdesk.Option("--password", "-p", dest="password", required=True),
superdesk.Option("--email", "-e", dest="email", required=True),
superdesk.Option("--admin", "-a", dest="admin", required=False, action="store_true"),
superdesk.Option("--support", "-s", dest="support", required=False, action="store_true"),
)
def run(self, username, password, email, admin=False, support=False):
# force type conversion to boolean
user_type = "administrator" if admin else "user"
userdata = {
"username": username,
"password": password,
"email": email,
"user_type": user_type,
"is_active": admin,
"is_support": support,
"needs_activation": not admin,
}
with app.test_request_context("/users", method="POST"):
if userdata.get("password", None) and not is_hashed(userdata.get("password")):
userdata["password"] = get_hash(
userdata.get("password"), app.config.get("BCRYPT_GENSALT_WORK_FACTOR", 12)
)
user = superdesk.get_resource_service("users").find_one(username=userdata.get("username"), req=None)
if user:
logger.info("user already exists %s" % (userdata))
else:
logger.info("creating user %s" % (userdata))
superdesk.get_resource_service("users").post([userdata])
logger.info("user saved %s" % (userdata))
return userdata
class ImportCommand(superdesk.Command):
"""Import articles into archives.
Example:
::
$ python manage.py xml:import ninjs data/sample.json
"""
option_list = [
superdesk.Option("parser", help="name of the feed parser"),
superdesk.Option("path", help="path to the archive file to parse"),
superdesk.Option("--profile",
"-p",
help="name of the profile to use (case sensitive"),
]
def run(self, parser, path, profile):
try:
feed_parser = registry.registered_feed_parsers[parser]
except KeyError:
print("Can't find feed parser with this name")
sys.exit(1)
if profile is not None:
content_types_service = get_resource_service("content_types")
try:
content_profile = content_types_service.find({
"label": profile
}).next()
except StopIteration:
print("Can't find content profile with this label")
sys.exit(1)
else:
profile_id = content_profile["_id"]
with open(path, "rb") as f:
buf = f.read()
buf = buf.replace(b"\r", b" ")
xml_parser = etree.XMLParser(recover=True)
parsed = etree.fromstring(buf, xml_parser)
articles = feed_parser.parse(parsed)
updates = {ITEM_STATE: "published"}
if profile is not None:
updates["profile"] = profile_id
for article in articles:
article.update(updates)
article.setdefault("source", parser)
archived_service = get_resource_service("archived")
archived_service.post(articles)
class DataUpdateCommand(superdesk.Command):
"""Parent class for Upgrade and Downgrade commands.
It defines options and initialize some variables in `run` method.
"""
option_list = [
superdesk.Option('--id', '-i', dest='data_update_id', required=False,
choices=get_data_updates_files(strip_file_extension=True),
help='Data update id to run last'),
superdesk.Option('--fake-init', dest='fake', required=False, action='store_true',
help='Mark data updates as run without actually running them'),
superdesk.Option('--dry-run', dest='dry', required=False, action='store_true',
help='Does not mark data updates as done. This can be useful for development.'),
]
def get_applied_updates(self):
req = ParsedRequest()
req.sort = '-name'
return tuple(self.data_updates_service.get(req=req, lookup={}))
def run(self, data_update_id=None, fake=False, dry=False):
self.data_updates_service = superdesk.get_resource_service('data_updates')
self.data_updates_files = get_data_updates_files(strip_file_extension=True)
# retrieve existing data updates in database
data_updates_applied = self.get_applied_updates()
self.last_data_update = data_updates_applied and data_updates_applied[-1] or None
if self.last_data_update:
if self.last_data_update['name'] not in self.data_updates_files:
print('A data update previously applied to this database (%s) can\'t be found in %s' % (
self.last_data_update['name'], ', '.join(get_dirs())))
def compile_update_in_module(self, data_update_name):
date_update_script_file = None
for folder in get_dirs():
date_update_script_file = os.path.join(folder, '%s.py' % (data_update_name))
if os.path.exists(date_update_script_file):
break
assert date_update_script_file is not None, 'File %s has not been found' % (data_update_name)
# create a module instance to use as scope for our data update
module = ModuleType('data_update_module')
with open(date_update_script_file) as f:
# compile data update script file
script = compile(f.read(), date_update_script_file, 'exec')
# excecute the script in the module
exec(script, module.__dict__)
return module
def in_db(self, update):
return update in map(lambda _: _['name'], self.get_applied_updates())
class SocketListenerCommand(superdesk.Command):
option_list = [
superdesk.Option('--address', '-a', dest='address', required=True),
superdesk.Option('--filter', '-f', dest='filter', required=True),
superdesk.Option('--start_message',
'-s',
dest='start_message',
required=True),
superdesk.Option('--end_message',
'-e',
dest='end_message',
required=True)
]
def run(self, address, filter, start_message, end_message):
print('Listening...')
asyncio.get_event_loop().run_until_complete(
self.listen(address, filter, start_message, end_message))
print('Stopped listening.')
@asyncio.coroutine
def listen(self, address, filter, start_message, end_message):
websocket = yield from websockets.connect(address)
start = 0
end = 0
try:
while True:
message = yield from websocket.recv()
if start_message and start_message in message:
start = time.time()
print("< {} at {}".format(
'Started', time.strftime("%H:%M:%S",
time.localtime())))
if filter and filter in message:
print("< {} at {}".format(
message, time.strftime("%H:%M:%S", time.localtime())))
if end_message and end_message in message:
end = time.time()
print("< {} at {}".format(
'Stopped', time.strftime("%H:%M:%S",
time.localtime())))
print("< Total time in seconds: %02d" % (end - start))
break
finally:
yield from websocket.close()
class CreateUserCommand(superdesk.Command):
"""Create a user with given username, password and email.
If user with given username exists, reset password.
"""
option_list = (
superdesk.Option('--username', '-u', dest='username', required=True),
superdesk.Option('--password', '-p', dest='password', required=True),
superdesk.Option('--email', '-e', dest='email', required=True),
superdesk.Option('--admin', '-a', dest='admin', required=False),
)
def run(self, username, password, email, admin='false'):
# force type conversion to boolean
is_admin = False if admin is None else json.loads(admin)
user_type = 'administrator' if is_admin else 'user'
userdata = {
'username': username,
'password': password,
'email': email,
'user_type': user_type,
'is_active': is_admin,
'needs_activation': not is_admin
}
with app.test_request_context('/users', method='POST'):
if userdata.get('password',
None) and not is_hashed(userdata.get('password')):
userdata['password'] = get_hash(
userdata.get('password'),
app.config.get('BCRYPT_GENSALT_WORK_FACTOR', 12))
user = superdesk.get_resource_service('users').find_one(
username=userdata.get('username'), req=None)
if user:
logger.info('updating user %s' % (userdata))
superdesk.get_resource_service('users').patch(
user.get('_id'), userdata)
return userdata
else:
logger.info('creating user %s' % (userdata))
superdesk.get_resource_service('users').post([userdata])
logger.info('user saved %s' % (userdata))
return userdata
class UpdateArchivedDocumentCommand(superdesk.Command):
"""
Update metadata for a document in both Mongodb and ElasticSearch by supplying ids
field to update and the value for the update
--field Field name
--value Value to be set in the field
--parseNeeded Optional. True if value is a complex type, not an int or string
Example:
::
$ manage.py app:updateArchivedDocument --ids='["588c1df11d41c80928015601","588c1b901d41c805dce70df0"]'
--field=anpa_category
--value=[{"scheme" : null,"qcode" : "f","subject" : "04000000","name" : "Finance"}]
--parseNeeded=True
"""
option_list = [
superdesk.Option('--ids', '-i', dest='ids', required=True),
superdesk.Option('--field', '-f', dest='field', required=True),
superdesk.Option('--value', '-v', dest='value', required=True),
superdesk.Option('--parseNeeded',
'-p',
dest='parseNeeded',
default=False)
]
def run(self, ids, field, value, parseNeeded=False):
ids = ast.literal_eval(ids)
if parseNeeded:
try:
value = json.loads(value)
except Exception as e:
print('Error in parsing the value: {}'.format(value))
print(e)
return
if ids and len(ids) > 0:
items = DeleteArchivedDocumentCommand().get_archived_items(ids)
for item in items:
superdesk.get_resource_service('archived').system_update(
bson.ObjectId(item['_id']), {field: value}, item)
print('Archived item {} has been updated.'.format(item['_id']))
print('-' * 45)
class ReassignMetadataCommand(superdesk.Command):
"""
Update metadata for a document in both Mongodb and ElasticSearch by supplying a GUID
manage.py app:update:metadata
-i urn:newsml:localhost:2016-09-12T12:11:40.160498:7237e59f-c42d-4865-aee5-e364aeb2966a
-m anpa_category
-v x
-m source -v AAP (value is a string name value)
-m anpa_category -v x (value is a qcode from vocabularies.json id=categories)
"""
option_list = [
superdesk.Option('--guid', '-i', dest='guids', required=True, action='append',
help='pass guids of articles to update a metadata item'),
superdesk.Option('--meta', '-m', dest='key', required=True),
superdesk.Option('--value', '-v', dest='value', required=True)
]
def run(self, guids, key, value):
category_list = superdesk.get_resource_service('vocabularies').find_one(req=None, _id='categories')
category_dict = {entry['qcode']: entry for entry in category_list['items']}
patched = value
if key not in ['anpa_category', 'source']:
print('Invalid meta value to update.')
sys.exit()
if key == 'anpa_category' and value not in category_dict:
print('Setting an invalid category metadata qcode.')
sys.exit()
if key == 'anpa_category':
entry = category_dict[value]
patched = [{
'qcode': entry.get('qcode'),
'name': entry.get('name'),
'subject': entry.get('subject', None),
'scheme': entry.get('scheme', None)
}]
service = superdesk.get_resource_service('archived')
docs = service.find({'guid': {'$in': guids}})
for original in docs:
res = service.system_update(original['_id'], {key: patched}, original)
print(res)
class RemoveExpiredContent(superdesk.Command):
"""Remove stale data from ingest based on the provider settings."""
option_list = (
superdesk.Option('--provider', '-p', dest='provider_name'),
)
def run(self, provider_name=None):
providers = list(superdesk.get_resource_service('ingest_providers').get(req=None, lookup={}))
self.remove_expired({'exclude': [str(p.get('_id')) for p in providers]})
for provider in providers:
if not provider_name or provider_name == provider.get('name'):
self.remove_expired(provider)
def remove_expired(self, provider):
lock_name = 'ingest:gc'
if not lock(lock_name, expire=300):
return
try:
remove_expired_data(provider)
push_notification('ingest:cleaned')
except Exception as err:
logger.exception(err)
raise ProviderError.expiredContentError(err, provider)
finally:
unlock(lock_name)
class UnregisterClient(superdesk.Command):
"""Remove a previously registered client
Example:
Unregister client with id ``0102030405060708090a0b0c``::
$ python manage.py auth_server:unregister_client 0102030405060708090a0b0c
"""
option_list = [
superdesk.Option('client_id'),
]
def run(self, client_id):
try:
client_id = ObjectId(client_id)
except InvalidId as e:
self.parser.error("the given client id is not valid: {msg}".format(msg=e))
clients_service = superdesk.get_resource_service('auth_server_clients')
client = clients_service.find_one(req=None, _id=client_id)
if client is None:
print("No client with id '{client_id}' found".format(client_id=client_id))
sys.exit(2)
clients_service.delete({'_id': client_id})
print("Client with id '{client_id}' has been successfuly unregistered".format(
client_id=client_id))
class AddProvider(superdesk.Command):
"""Add ingest provider."""
option_list = {
superdesk.Option('--provider', '-p', dest='provider'),
}
def run(self, provider=None):
if provider:
try:
data = {}
data = superdesk.json.loads(provider)
data.setdefault('content_expiry', superdesk.app.config['INGEST_EXPIRY_MINUTES'])
validator = superdesk.app.validator(superdesk.app.config['DOMAIN']['ingest_providers']['schema'],
'ingest_providers')
validation = validator.validate(data)
if validation:
get_resource_service('ingest_providers').post([data])
return data
else:
ex = Exception('Failed to add Provider as the data provided is invalid. Errors: {}'
.format(str(validator.errors)))
raise ProviderError.providerAddError(exception=ex, provider=data)
except Exception as ex:
raise ProviderError.providerAddError(ex, data)
class RebuildElasticIndex(superdesk.Command):
"""Rebuild the elastic indexes from existing data.
It creates new index with same alias as the configured index,
puts the new mapping and removes the old index.
Example:
::
$ python manage.py app:rebuild_elastic_index
$ python manage.py app:rebuild_elastic_index --resource=items
$ python manage.py app:rebuild_elastic_index --resource=archive
"""
option_list = [superdesk.Option('--resource', '-r', dest='resource_name')]
def run(self, resource_name=None):
# if no index name is passed then use the configured one
resources = list(
current_app.data.elastic._get_elastic_resources().keys())
if resource_name and resource_name in resources:
resources = [resource_name]
elif resource_name:
raise ValueError(
"Resource {} is not configured".format(resource_name))
for resource in resources:
try:
current_app.data.elastic.reindex(resource)
except elasticsearch.exceptions.NotFoundError as nfe:
print(nfe)
print('Index {} rebuilt successfully.'.format(resource_name))
class AppPrepopulateCommand(superdesk.Command):
option_list = [
superdesk.Option('--file',
'-f',
dest='prepopulate_file',
default='app_prepopulate_data.json'),
superdesk.Option('--dir', '-d', dest='directory', default=None),
]
def run(self, prepopulate_file, directory=None):
user = get_resource_service('users').find_one(
username=get_default_user()['username'], req=None)
if not user:
get_resource_service('users').post([get_default_user()])
prepopulate_data(prepopulate_file, get_default_user(), directory)
class ImportLegalPublishQueueCommand(superdesk.Command):
"""
This command import publish queue records into legal publish queue.
Example:
::
$ python manage.py legal_publish_queue:import
$ python manage.py legal_publish_queue:import --page-size=100
"""
default_page_size = 500
option_list = [
superdesk.Option('--page-size', '-p', dest='page_size', required=False)
]
def run(self, page_size=None):
if not is_legal_archive_enabled():
return
logger.info('Import to Legal Publish Queue')
lock_name = get_lock_id('legal_archive', 'import_legal_publish_queue')
page_size = int(page_size) if page_size else self.default_page_size
if not lock(lock_name, expire=310):
return
try:
LegalArchiveImport().import_legal_publish_queue(page_size=page_size)
finally:
unlock(lock_name)
class ImportUserProfileFromADCommand(superdesk.Command):
"""
Responsible for importing a user profile from Active Directory (AD) to Mongo.
This command runs on assumption that the user executing this command and
the user whose profile need to be imported need not to be the same. Uses ad_username and ad_password to bind to AD
and then searches for a user identified by username_to_import and if found imports into Mongo.
"""
option_list = (
superdesk.Option('--ad_username', '-adu', dest='ad_username', required=True),
superdesk.Option('--ad_password', '-adp', dest='ad_password', required=True),
superdesk.Option('--username_to_import', '-u', dest='username', required=True),
superdesk.Option('--admin', '-a', dest='admin', required=False),
)
def run(self, ad_username, ad_password, username, admin='false'):
"""
Imports or Updates a User Profile from AD to Mongo.
:param ad_username: Active Directory Username
:param ad_password: Password of Active Directory Username
:param username: Username as in Active Directory whose profile needs to be imported to Superdesk.
:return: User Profile.
"""
# force type conversion to boolean
user_type = 'administrator' if admin is not None and admin.lower() == 'true' else 'user'
# Authenticate and fetch profile from AD
settings = app.settings
ad_auth = ADAuth(settings['LDAP_SERVER'], settings['LDAP_SERVER_PORT'], settings['LDAP_BASE_FILTER'],
settings['LDAP_USER_FILTER'], settings['LDAP_USER_ATTRIBUTES'], settings['LDAP_FQDN'])
user_data = ad_auth.authenticate_and_fetch_profile(ad_username, ad_password, username)
if len(user_data) == 0:
raise SuperdeskApiError.notFoundError('Username not found')
# Check if User Profile already exists in Mongo
user = superdesk.get_resource_service('users').find_one(username=username, req=None)
if user:
superdesk.get_resource_service('users').patch(user.get('_id'), user_data)
else:
add_default_values(user_data, username, user_type=user_type)
superdesk.get_resource_service('users').post([user_data])
return user_data
class RemoveExportedFiles(superdesk.Command):
"""Remove files from storage that were used for exporting items
Example:
::
$ python manage.py storage:remove_exported
$ python manage.py storage:remove_exported --expire-hours=12
"""
log_msg = ""
expire_hours = 24
option_list = [
superdesk.Option("--expire-hours",
"-e",
dest="expire_hours",
required=False,
type=int)
]
def run(self, expire_hours=None):
if expire_hours:
self.expire_hours = expire_hours
elif "TEMP_FILE_EXPIRY_HOURS" in app.config:
self.expire_hours = app.config["TEMP_FILE_EXPIRY_HOURS"]
expire_at = utcnow() - timedelta(hours=self.expire_hours)
self.log_msg = "Expiry Time: {}.".format(expire_at)
logger.info("{} Starting to remove exported files from storage".format(
self.log_msg))
lock_name = get_lock_id("storage", "remove_exported")
if not lock(lock_name, expire=300):
logger.info(
"Remove exported files from storage task is already running")
return
try:
logger.info("{} Removing expired temporary media files".format(
self.log_msg))
self._remove_exported_files(expire_at)
finally:
unlock(lock_name)
logger.info("{} Completed removing exported files from storage".format(
self.log_msg))
def _remove_exported_files(self, expire_at):
logger.info(
"{} Beginning to remove exported files from storage".format(
self.log_msg))
for file_id in self._get_file_ids(expire_at):
app.media.delete(file_id)
def _get_file_ids(self, expire_at):
files = app.media.find(folder="temp", upload_date={"$lte": expire_at})
return [file["_id"] for file in files]
class RemoveExportedFiles(superdesk.Command):
"""Remove files from storage that were used for exporting items
Example:
::
$ python manage.py storage:remove_exported
$ python manage.py storage:remove_exported --expire-hours=12
"""
log_msg = ''
expire_hours = 24
option_list = [
superdesk.Option('--expire-hours',
'-e',
dest='expire_hours',
required=False,
type=int)
]
def run(self, expire_hours=None):
if expire_hours:
self.expire_hours = expire_hours
elif 'TEMP_FILE_EXPIRY_HOURS' in app.config:
self.expire_hours = app.config['TEMP_FILE_EXPIRY_HOURS']
expire_at = utcnow() - timedelta(hours=self.expire_hours)
self.log_msg = 'Expiry Time: {}.'.format(expire_at)
logger.info('{} Starting to remove exported files from storage'.format(
self.log_msg))
lock_name = get_lock_id('storage', 'remove_exported')
if not lock(lock_name, expire=300):
logger.info(
'Remove exported files from storage task is already running')
return
try:
logger.info('{} Removing expired temporary media files'.format(
self.log_msg))
self._remove_exported_files(expire_at)
finally:
unlock(lock_name)
logger.info('{} Completed removing exported files from storage'.format(
self.log_msg))
def _remove_exported_files(self, expire_at):
logger.info(
'{} Beginning to remove exported files from storage'.format(
self.log_msg))
for file_id in self._get_file_ids(expire_at):
app.media.delete(file_id)
def _get_file_ids(self, expire_at):
files = app.media.find(folder='temp', upload_date={'$lte': expire_at})
return [file['_id'] for file in files]
class IndexFromMongo(superdesk.Command):
"""
Index the specified mongo collection in the specified elastic collection/type.
This will use the default APP mongo DB to read the data and the default Elastic APP index.
"""
option_list = [
superdesk.Option('--from',
'-f',
dest='mongo_collection_name',
required=True),
superdesk.Option('--page-size', '-p', dest='page_size')
]
default_page_size = 500
def run(self, mongo_collection_name, page_size):
bucket_size = int(page_size) if page_size else self.default_page_size
print('Indexing data from mongo/{} to elastic/{}'.format(
mongo_collection_name, mongo_collection_name))
service = superdesk.get_resource_service(mongo_collection_name)
cursor = service.get_from_mongo(None, {})
count = cursor.count()
no_of_buckets = len(range(0, count, bucket_size))
print('Number of items to index: {}, pages={}'.format(
count, no_of_buckets))
for x in range(0, no_of_buckets):
skip = x * bucket_size
print('Page : {}, skip: {}'.format(x + 1, skip))
cursor = service.get_from_mongo(None, {})
cursor.skip(skip)
cursor.limit(bucket_size)
items = list(cursor)
print('Inserting {} items'.format(len(items)))
success, failed = superdesk.app.data._search_backend(
mongo_collection_name).bulk_insert(mongo_collection_name,
items)
print('Inserted {} items'.format(success))
if failed:
print(
'Failed to do bulk insert of items {}. Errors: {}'.format(
len(failed), failed))
raise BulkIndexError(resource=mongo_collection_name,
errors=failed)
return 'Finished indexing collection {}'.format(mongo_collection_name)
