def get_options(self):
options = (
Option(
'-e',
'--email',
dest='email',
nargs='*',
help='One or more email addresses to send a test email to.'),
Option(
'-m',
'--managers',
dest='managers',
action='store_true',
help=
'Send a test email to the addresses specified in settings.MANAGERS.'
),
Option(
'-a',
'--admins',
dest='admins',
action='store_true',
help=
'Send a test email to the addresses specified in settings.ADMINS.'
),
)
return options
def get_options(self):
options = (
Option('name', help='Name of the application or project.'),
Option('directory',
nargs='?',
help='Optional destination directory'),
Option('--template',
help='The path or URL to load the template from.'),
Option('--extension',
'-e',
dest='extensions',
action='append',
default=['py'],
help='The file extension(s) to render (default: "py"). '
'Separate multiple extensions with commas, or use '
'-e multiple times.'),
Option(
'--name',
'-n',
dest='files',
action='append',
default=[],
help='The file name(s) to render. Separate multiple file names '
'with commas, or use -n multiple times.'))
return options
def get_options(self):
options = (
Option(
'--locale',
'-l',
action='append',
default=[],
help=
'Locale(s) to process (e.g. de_AT). Default is to process all. '
'Can be used multiple times.'),
Option(
'--exclude',
'-x',
action='append',
default=[],
help=
'Locales to exclude. Default is none. Can be used multiple times.'
),
Option('--use-fuzzy',
'-f',
dest='fuzzy',
action='store_true',
help='Use fuzzy translations.'),
)
return options
def get_options(self):
options = super().get_options()
options += (
Option('-h',
'--host',
default=self.default_host,
help='Server hostname.'),
Option('-p', '--port', required=True, help='Server port number.'),
)
return options
def get_options(self):
return (
Option('--no-ipython', action="store_true", dest='no_ipython', default=(not self.use_ipython),
help="Do not use the IPython shell"),
Option('--no-bpython', action="store_true", dest='no_bpython', default=(not self.use_bpython),
help="Do not use the BPython shell"),
Option('--no-ptipython', action="store_true", dest='no_ptipython', default=(not self.use_ptipython),
help="Do not use the PtIPython shell"),
Option('--no-ptpython', action="store_true", dest='no_ptpython', default=(not self.use_ptpython),
help="Do not use the PtPython shell"),
)
class ApplicationChooser(Command):
help = description = 'Choose application for administration.'
option_list = (
Option('name', help='Name of the application.'),
)
def run(self, *args, **kwargs):
pass
class GeoIPMMDBUpdate(Command):
help = description = 'Creates or updates geoip2 mmdb databases.'
option_list = (Option('-p',
'--path',
default='',
help='Path where files to save'), )
def run(self, path):
update(path)
class DumpData(Command):
help = description = 'Output the contents of the database as a fixture of the given format.'
option_list = (
Option('args',
metavar='model_name',
nargs='*',
help='Restricts dumped data to the specified model name.'),
Option('--format',
default='json',
help='Specifies the output serialization format for fixtures.'),
Option(
'--indent',
type=int,
help=
'Specifies the indent level to use when pretty-printing output.'),
Option('-e',
'--exclude',
action='append',
default=[],
help='A model name to exclude '
'(use multiple --exclude to exclude multiple models).'),
Option(
'--pks',
dest='primary_keys',
help=
'Only dump objects with given primary keys. Accepts a comma-separated '
'list of keys. This option only works when you specify one model.'
),
Option('-o',
'--output',
help='Specifies file to which the output is written.'))
# noinspection PyAttributeOutsideInit
def run(self, *model_names, **options):
self.format = options['format']
self.indent = options['indent']
self.excluded_models = options['exclude']
self.output = options['output']
pks = options['primary_keys']
if pks:
self.primary_keys = [pk.strip() for pk in pks.split(',')]
else:
self.primary_keys = []
from anthill.framework.apps.builder import app
if not model_names:
if self.primary_keys:
raise InvalidCommand(
"You can only use --pks option with one model")
self.models = list(
filter(lambda m: m.__name__ not in self.excluded_models,
app.get_models()))
else:
if len(model_names) > 1 and self.primary_keys:
raise InvalidCommand(
"You can only use --pks option with one model")
self.models = []
for model_name in model_names:
model = app.get_model(model_name)
if model is None:
raise InvalidCommand("Unknown model: %s" % model_name)
self.models.append(model)
try:
self.stdout.ending = None
progress_output = None
object_count = 0
# If dumpdata is outputting to stdout, there is no way to display progress
if self.output and self.stdout.isatty():
progress_output = self.stdout
object_count = sum(self.get_objects(count_only=True))
stream = open(self.output, 'w') if self.output else None
try:
serialize(self.format,
self.get_objects(),
indent=indent,
stream=stream or self.stdout,
progress_output=progress_output,
object_count=object_count)
finally:
if stream:
stream.close()
except Exception as e:
raise InvalidCommand("Unable to serialize database: %s" % e)
def get_objects(self, count_only=False):
"""
Collate the objects to be serialized. If count_only is True, just
count the number of objects to be serialized.
"""
for model in self.models:
if model in excluded_models:
continue
query = Model.query
if self.primary_keys:
query = query.filter(Model.id.in_(self.primary_keys))
if count_only:
yield query.count()
else:
yield query.all()
class ReplaceCommand(Command):
help = 'Perform replace operations.'
name = 'replace'
option_list = (
Option('-f', '--file', dest='file', default='replaces.json',
help='JSON file with a list of replace pairs.'),
Option('-t', '--target', dest='target', default='$',
help='Target path of json tree.'),
Option('-u', '--users', dest='users', default=None,
help='User id list separated by commas.'),
)
@staticmethod
def _load_replaces(path: str) -> Dict[Any, Any]:
"""Load replace pairs from JSON file."""
with open(path) as f:
replaces = json.load(f)
return dict(replaces)
def get_replaces(self, path: str) -> Dict[Any, Any]:
try:
replaces = self._load_replaces(path)
if not replaces:
self.stdout.write('No replaces to perform.')
sys.exit()
except (FileNotFoundError, JSONDecodeError) as e:
self.stderr.write(str(e))
sys.exit(GET_REPLACES_ERROR)
return replaces
@staticmethod
def parse_users(raw_users: Optional[str] = None) -> List[str]:
"""
Parse string of user ids list separated by commas.
:return: user ids list.
"""
if raw_users is not None:
return re.split(r'\s*,\s*', raw_users)
return []
@staticmethod
def get_profiles(users: Optional[List[str]] = None) -> List[Profile]:
"""Load profiles from database."""
query = Profile.query
if users:
query = query.filter(Profile.user_id.in_(users))
return query.all()
@staticmethod
def replace(profile: Profile, target: str, replaces: Dict[Any, Any]) -> None:
"""Apply all replace operations on profile payload."""
matches = profile.find_payload(target, lambda x: x.value in replaces)
for match in matches:
new_value = replaces[match.value]
# do replace operation without committing to database
profile.update_payload(match.full_path, new_value, commit=False)
if matches:
# finally commit changes to database
profile.save()
def run(self, file: str, target: str, users: Optional[str] = None) -> None:
replaces = self.get_replaces(path=file)
profiles = self.get_profiles(users=self.parse_users(users))
with tqdm.tqdm(total=len(profiles), unit=' profile') as pb:
for profile in profiles:
self.replace(profile, target, replaces)
pb.update()
class LoadData(Command):
help = description = 'Installs the named fixture(s) in the database.'
option_list = (
Option('args', metavar='fixture', nargs='+', help='Fixture labels.'),
Option(
'--ignorenonexistent',
'-i',
action='store_true',
dest='ignore',
help='Ignores entries in the serialized data for fields that do not '
'currently exist on the model.'),
Option('-e',
'--exclude',
action='append',
default=[],
help='A model name to exclude. Can be used multiple times.'),
Option('--format',
help='Format of serialized data when reading from stdin.'),
)
# noinspection PyAttributeOutsideInit
def run(self, *fixture_labels, **options):
self.ignore = options['ignore']
self.excluded_models = options['exclude']
self.format = options['format']
self.loaddata(fixture_labels)
# noinspection PyAttributeOutsideInit
def loaddata(self, fixture_labels):
# Keep a count of the installed objects and fixtures
self.fixture_count = 0
self.loaded_object_count = 0
self.fixture_object_count = 0
self.models = set()
self.serialization_formats = get_deserializers(keys=True)
self.compression_formats = {
None: (open, 'rb'),
'gz': (gzip.GzipFile, 'rb'),
'zip': (SingleZipReader, 'r'),
'stdin': (lambda *args: sys.stdin, None),
}
if has_bz2:
self.compression_formats['bz2'] = (bz2.BZ2File, 'r')
# Anthill's test suite repeatedly tries to load initial_data fixtures
# from apps that don't have any fixtures. Because disabling constraint
# checks can be expensive on some database (especially MSSQL), bail
# out early if no fixtures are found.
for fixture_label in fixture_labels:
if self.find_fixtures(fixture_label):
break
else:
return
for fixture_label in fixture_labels:
self.load_label(fixture_label)
if self.fixture_object_count == self.loaded_object_count:
self.stdout.write("Installed %d object(s) from %d fixture(s)" %
(self.loaded_object_count, self.fixture_count))
else:
self.stdout.write(
"Installed %d object(s) (of %d) from %d fixture(s)" %
(self.loaded_object_count, self.fixture_object_count,
self.fixture_count))
def load_label(self, fixture_label):
"""Load fixtures files for a given label."""
for fixture_file, fixture_dir, fixture_name in self.find_fixtures(
fixture_label):
_, ser_fmt, cmp_fmt = self.parse_name(
os.path.basename(fixture_file))
open_method, mode = self.compression_formats[cmp_fmt]
fixture = open_method(fixture_file, mode)
try:
self.fixture_count += 1
objects_in_fixture = 0
loaded_objects_in_fixture = 0
self.stdout.write(
"Installing %s fixture '%s' from %s." %
(ser_fmt, fixture_name, humanize(fixture_dir)))
objects = deserialize(ser_fmt,
fixture,
ignorenonexistent=self.ignore)
for obj in objects:
objects_in_fixture += 1
if obj.__class__.__name__ in self.excluded_models:
continue
loaded_objects_in_fixture += 1
self.models.add(obj.__class__)
try:
obj.save()
self.stdout.write('\rProcessed %i object(s).' %
loaded_objects_in_fixture,
ending='')
except Exception as e:
e.args = (
"Could not load %(class_name)s(id=%(id)s): %(error_msg)s"
% {
'class_name': obj.__class__.__name__,
'id': obj.id,
'error_msg': e,
}, )
raise
if objects:
self.stdout.write(
'') # add a newline after progress indicator
self.loaded_object_count += loaded_objects_in_fixture
self.fixture_object_count += objects_in_fixture
except Exception as e:
if not isinstance(e, InvalidCommand):
e.args = ("Problem installing fixture '%s': %s" %
(fixture_file, e), )
raise
finally:
fixture.close()
# Warn if the fixture we loaded contains 0 objects.
if objects_in_fixture == 0:
warnings.warn(
"No fixture data found for '%s'. (File format may be "
"invalid.)" % fixture_name, RuntimeWarning)
@functools.lru_cache(maxsize=None)
def find_fixtures(self, fixture_label):
"""Find fixture files for a given label."""
if fixture_label == READ_STDIN:
return [(READ_STDIN, None, READ_STDIN)]
fixture_name, ser_fmt, cmp_fmt = self.parse_name(fixture_label)
cmp_fmts = list(
self.compression_formats) if cmp_fmt is None else [cmp_fmt]
ser_fmts = self.serialization_formats if ser_fmt is None else [ser_fmt]
self.stdout.write("Loading '%s' fixtures..." % fixture_name)
if os.path.isabs(fixture_name):
fixture_dirs = [os.path.dirname(fixture_name)]
fixture_name = os.path.basename(fixture_name)
else:
fixture_dirs = self.fixture_dirs
if os.path.sep in os.path.normpath(fixture_name):
fixture_dirs = [
os.path.join(dir_, os.path.dirname(fixture_name))
for dir_ in fixture_dirs
]
fixture_name = os.path.basename(fixture_name)
suffixes = ('.'.join(ext for ext in combo if ext)
for combo in product(ser_fmts, cmp_fmts))
targets = {'.'.join((fixture_name, suffix)) for suffix in suffixes}
fixture_files = []
for fixture_dir in fixture_dirs:
self.stdout.write("Checking %s for fixtures..." %
humanize(fixture_dir))
fixture_files_in_dir = []
path = os.path.join(fixture_dir, fixture_name)
for candidate in glob.iglob(glob.escape(path) + '*'):
if os.path.basename(candidate) in targets:
# Save the fixture_dir and fixture_name for future error messages.
fixture_files_in_dir.append(
(candidate, fixture_dir, fixture_name))
if not fixture_files_in_dir:
self.stdout.write("No fixture '%s' in %s." %
(fixture_name, humanize(fixture_dir)))
# Check kept for backwards-compatibility; it isn't clear why
# duplicates are only allowed in different directories.
if len(fixture_files_in_dir) > 1:
raise InvalidCommand(
"Multiple fixtures named '%s' in %s. Aborting." %
(fixture_name, humanize(fixture_dir)))
fixture_files.extend(fixture_files_in_dir)
if not fixture_files:
raise InvalidCommand("No fixture named '%s' found." % fixture_name)
return fixture_files
@cached_property
def fixture_dirs(self):
"""Return a list of fixture directories."""
dirs = []
fixture_dirs = settings.FIXTURE_DIRS
if len(fixture_dirs) != len(set(fixture_dirs)):
raise ImproperlyConfigured(
"settings.FIXTURE_DIRS contains duplicates.")
dirs.extend(fixture_dirs)
dirs.append('')
return [os.path.realpath(d) for d in dirs]
def parse_name(self, fixture_name):
"""
Split fixture name in name, serialization format, compression format.
"""
if fixture_name == READ_STDIN:
if not self.format:
raise InvalidCommand(
'--format must be specified when reading from stdin.')
return READ_STDIN, self.format, 'stdin'
parts = fixture_name.rsplit('.', 2)
if len(parts) > 1 and parts[-1] in self.compression_formats:
cmp_fmt = parts[-1]
parts = parts[:-1]
else:
cmp_fmt = None
if len(parts) > 1:
if parts[-1] in self.serialization_formats:
ser_fmt = parts[-1]
parts = parts[:-1]
else:
raise InvalidCommand(
"Problem installing fixture '%s': %s is not a known "
"serialization format." %
('.'.join(parts[:-1]), parts[-1]))
else:
ser_fmt = None
name = '.'.join(parts)
return name, ser_fmt, cmp_fmt