def add_arguments(self, parser: CommandParser):
# noinspection PyTypeChecker
parser.add_argument("input", type=Path, help="Path to the json file")
parser.add_argument("--ags", help="The Amtliche Gemeindeschlüssel")
parser.add_argument(
"--skip-download",
action="store_true",
dest="skip_download",
default=False,
help="Do not download and parse the files",
)
parser.add_argument(
"--skip-body-extra",
action="store_true",
dest="skip_body_extra",
default=False,
help="Do not download streets and shape of the body",
)
parser.add_argument(
"--allow-shrinkage",
action="store_true",
dest="allow_shrinkage",
default=False,
help=
"Don't fail when trying to import a smaller dataset over a bigger existing one",
)
def get_local_test_settings_file(argv):
assert argv[1] == "test"
assert "manage.py" in argv[0]
local_settings_dir = os.path.join(
os.path.split(argv[0])[0], "local_settings")
from django.core.management import CommandParser, CommandError
parser = CommandParser(usage="%(prog)s subcommand [options] [args]",
add_help=False)
parser.add_argument('--local_test_settings', dest="local_test_settings")
options, args = parser.parse_known_args(argv)
if options.local_test_settings is None:
local_settings_file = "local_settings_example.py"
else:
local_settings_file = options.local_test_settings
if os.path.split(local_settings_file)[0] == "":
local_settings_file = os.path.join(local_settings_dir,
local_settings_file)
if os.path.abspath(local_settings_file) == os.path.abspath(
os.path.join(local_settings_dir, "local_settings.py")):
raise CommandError("Using production local_settings for tests is not "
"allowed due to security reason.")
if not os.path.isfile(local_settings_file):
raise CommandError("file '%s' does not exist" % local_settings_file)
return local_settings_file
def add_arguments(self, parser: CommandParser):
parser.add_argument(
'table',
type=str,
help=
'A table to clear all data from. Options: blocks, transactions, actions'
)
def create_parser(self, prog_name, subcommand):
"""
Create and return the ``ArgumentParser`` which will be used to
parse the arguments to this command.
"""
print(self, "%s %s" % (os.path.basename(prog_name), subcommand), self.help or None)
parser = CommandParser(
self,
prog="%s %s" % (os.path.basename(prog_name), subcommand),
description=self.help or None,
)
parser.add_argument(
"-v",
"--verbosity",
action="store",
dest="verbosity",
default=1,
type=int,
choices=[0, 1, 2, 3],
help="Verbosity level; 0=minimal output, 1=normal output, 2=verbose output, 3=very verbose output",
)
parser.add_argument(
"--no-color",
action="store_true",
dest="no_color",
default=False,
help="Don't colorize the command output.",
)
self.add_arguments(parser)
return parser
def add_arguments(self, parser: CommandParser) -> None:
"""
:param parser: CommandParser instance to add the runtime argument
:return: None
"""
parser.add_argument("--count",
type=int,
help="Number of endpoint names to be generated")
def add_arguments(self, parser: CommandParser):
parser.add_argument(
'-f',
'--force',
action='store_true',
dest='force',
default=False,
help=
'Do not show warning / ask if you are sure before deleting ALL locks'
)
def execute_from_command_line(argv=None):
from security.command import CommandExecutor
def execute_from_command_line_with_stdout(argv=None,
stdout=None,
stderr=None):
try:
from django.core.management.base import BaseCommand
command_stdout = stdout
command_stderr = stderr
def command_init_patch(self,
stdout=None,
stderr=None,
no_color=False,
force_color=False):
stdout = command_stdout if stdout is None else stdout
stderr = command_stderr if stderr is None else stderr
self._patched_init(stdout=stdout,
stderr=stderr,
no_color=no_color,
force_color=force_color)
BaseCommand._patched_init = BaseCommand.__init__
BaseCommand.__init__ = command_init_patch
execute_from_command_line_original(argv=argv)
finally:
BaseCommand.__init__ = BaseCommand._patched_init
if len(argv) > 1:
# some arguments must be processed before django setup
parser_args = (None, ) if StrictVersion(
get_main_version()) < StrictVersion('2.1') else tuple()
parser = CommandParser(*parser_args,
usage='%(prog)s subcommand [options] [args]',
add_help=False)
parser.add_argument('--settings')
parser.add_argument('--pythonpath')
parser.add_argument('args', nargs='*') # catch-all
try:
options, args = parser.parse_known_args(argv[2:])
handle_default_options(options)
except CommandError:
pass # Ignore any option errors at this point.
django.setup()
return CommandExecutor(
command_function=execute_from_command_line_with_stdout,
command_kwargs={
'argv': argv
},
name=argv[1],
input=' '.join(argv[2:]),
is_executed_from_command_line=True).run()
else:
execute_from_command_line_original(argv=argv)
def add_arguments(self, parser: CommandParser):
invitation = parser.add_argument_group(
_("Event"), _("Create an link imported guests to an event"))
invitation.add_argument("--date",
dest="event_date",
type=parse_date,
help=_("date of the event"))
invitation.add_argument("--name",
dest="event_name",
type=str,
help=_("name of the event"))
parser.add_argument("csv", help=_("path to the csv file to parse"))
def add_arguments(self, parser: CommandParser):
parser.add_argument('blocks',
type=int,
nargs='+',
help='One or more blocks to (re-)import')
parser.add_argument(
'-f',
'--force',
help='If the block(s) already exist, delete and re-import them.',
action='store_true',
dest='force',
default=False)
def add_arguments(self, parser: CommandParser):
"""Additional custom arguments"""
super(Command, self).add_arguments(parser)
parser.add_argument(
'--node_type', choices=[CONFIRMATION_VALIDATOR, PRIMARY_VALIDATOR])
parser.add_argument(
'--seed_block_identifier',
type=str_length_validator(length=BLOCK_IDENTIFIER_LENGTH))
parser.add_argument('--head_block_hash',
type=str_length_validator(length=HEAD_HASH_LENGTH))
parser.add_argument('--root_account_file',
type=url_validator(suffix='.json'))
parser.add_argument('--daily_confirmation_rate', type=int)
def add_arguments(self, parser: CommandParser):
parser.add_argument(
"--queue",
default="batch2",
help="The celery queue where the tasks should be processed.",
)
parser.add_argument(
"--courts",
type=str,
default=["all"],
nargs="*",
help="The courts that you wish to parse.",
)
parser.add_argument(
"--iterations",
type=int,
default=0,
help="The number of iterations to take. Default is 0, which means "
"to loop forever",
)
parser.add_argument(
"--iteration-delay",
type=float,
default=1.0,
help="How long to wait after completing an iteration of all "
"courts before beginning another iteration",
)
def add_arguments(self, parser: CommandParser) -> None:
class SubCommandParser(CommandParser):
def __init__(self, **kwargs: Any) -> None:
super().__init__(**kwargs)
subparsers = parser.add_subparsers(
dest='scmd',
title="subcommands",
parser_class=SubCommandParser
) # type: argparse._SubParsersAction
inst_sp = subparsers.add_parser("install", help="Create stored procedure API in the database")
grant_sp = subparsers.add_parser("grant", help="Grant access to the stored procedure API to a database user")
grant_sp.add_argument(
"user", nargs="+",
help="The names of the database users to grant access to the stored procedures. 'install' must have been "
"called before and the users must already have been created in PostgreSQL"
)
revoke_sp = subparsers.add_parser("revoke", help="Revoke access to the stored procedure API from a database "
"user")
revoke_sp.add_argument("user", nargs="+", action="append", default=[],
help="The names of the database users to revoke access from")
check_sp = subparsers.add_parser("check", help="Check whether the spapi is installed or a user as access")
check_sp_g = check_sp.add_mutually_exclusive_group(required=True)
check_sp_g.add_argument("--installed", dest="check_installed", action="store_true", default=False)
check_sp_g.add_argument("--grant", dest="user", type=str)
def add_arguments(self, parser: CommandParser):
parser.add_argument(
'--start-block',
type=int,
help=
'Start from this block (note --start-type affects meaning of this)',
dest='start_block',
default=None)
parser.add_argument(
'--start-type',
type=str,
help=
"Either 'rel' (start block means relative blocks behind head),\n"
"or 'exact' (start block means start from this exact block number)",
dest='start_type',
default=None)
def add_arguments(self, parser: CommandParser):
parser.add_argument('plugins', nargs='+', type=str)
parser.add_argument('--upgrade',
action='store_true',
dest='upgrade',
help='Upgrade plugins')
parser.add_argument('--clear',
action='store_true',
dest='clear',
help='Clear plugins')
parser.add_argument('--validate',
action='store_true',
dest='validate',
help='Validate plugins')
parser.add_argument('--index',
action='store_true',
dest='index',
help='Index plugins')
def execute_from_command_line(argv=None):
def execute_from_command_line_with_stdout(argv=None,
stdout=None,
stderr=None):
try:
if stdout:
sys.stdout = stdout
if stderr:
sys.stderr = stderr
execute_from_command_line_original(argv=argv)
finally:
if stdout:
sys.stdout = sys.__stdout__
if stderr:
sys.stderr = sys.__stderr__
if len(argv) > 1:
from security.utils import CommandLogger
# some arguments must be processed before django setup
parser_args = (None, ) if StrictVersion(
get_main_version()) < StrictVersion('2.1') else tuple()
parser = CommandParser(*parser_args,
usage='%(prog)s subcommand [options] [args]',
add_help=False)
parser.add_argument('--settings')
parser.add_argument('--pythonpath')
parser.add_argument('args', nargs='*') # catch-all
try:
options, args = parser.parse_known_args(argv[2:])
handle_default_options(options)
except CommandError:
pass # Ignore any option errors at this point.
django.setup()
return CommandLogger(
command_function=execute_from_command_line_with_stdout,
command_kwargs={
'argv': argv
},
name=argv[1],
input=' '.join(argv[2:]),
executed_from_command_line=True).run()
else:
execute_from_command_line_original(argv=argv)
def add_arguments(self, parser: CommandParser):
parser.add_argument(
'-s',
'--store',
type=AppStoreType,
default=AppStore.objects.first(),
help="""
Output charts for a specific store. Since in most cases only a
single store is present, the first one found in the database is
used by default. The store is specified by the country code,
e. g., us or de.
""",
)
parser.add_argument(
'app',
type=int,
help="""
The ID of the application, for which metadata should be returned.
""",
)
def create_parser(self, prog_name, subcommand):
"""
Create and return the ``ArgumentParser`` which will be used to
parse the arguments to this command.
"""
print(self, "%s %s" % (os.path.basename(prog_name), subcommand), self.help
or None)
parser = CommandParser(
self,
prog="%s %s" % (os.path.basename(prog_name), subcommand),
description=self.help or None,
)
parser.add_argument(
"-v",
"--verbosity",
action="store",
dest="verbosity",
default=1,
type=int,
choices=[0, 1, 2, 3],
help=
"Verbosity level; 0=minimal output, 1=normal output, 2=verbose output, 3=very verbose output",
)
parser.add_argument(
"--no-color",
action="store_true",
dest="no_color",
default=False,
help="Don't colorize the command output.",
)
self.add_arguments(parser)
return parser
def add_arguments(self, parser: CommandParser) -> None:
parser.add_argument(
'app_label',
nargs='?',
type=str,
help=
'Django App name to migrate. By default all found apps are migrated.'
)
parser.add_argument(
'migration_number',
nargs='?',
type=int,
help='Migration number in selected django app to migrate to.'
' By default all available migrations are applied.'
' Note that library currently have no ability rollback migrations')
parser.add_argument(
'--database',
'-d',
nargs='?',
type=str,
required=False,
choices=list(config.DATABASES.keys()),
help=
'ClickHouse database alias key from CLICKHOUSE_DATABASES django setting.'
' By default migrations are applied to all databases.')
def main():
"""Run administrative tasks."""
parser = CommandParser()
parser.add_argument("--env")
try:
env_option, rest = parser.parse_known_args(sys.argv)
sys.argv = rest
os.environ["env"] = env_option.env
except:
pass
os.environ.setdefault("DJANGO_SETTINGS_MODULE",
"secrets_manager_prototype.settings")
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?") from exc
execute_from_command_line(sys.argv)
def create_parser(self, prog_name, subcommand):
"""
Override in order to skip default parameters like verbosity, version, etc.
"""
parser = CommandParser(
self,
prog="%s %s" % (os.path.basename(prog_name), subcommand),
description=self.help or None,
)
# create hidden options (required by BaseCommand)
parser.add_argument('--no-color', help=argparse.SUPPRESS)
parser.add_argument('--pythonpath', help=argparse.SUPPRESS)
parser.add_argument('--traceback', help=argparse.SUPPRESS)
self.add_arguments(parser)
return parser
def get_local_test_settings_file(argv):
assert argv[1] == "test"
assert "manage.py" in argv[0]
local_settings_dir = os.path.split(argv[0])[0]
assert os.path.isfile(os.path.join(local_settings_dir, "manage.py"))
from django.core.management import CommandParser, CommandError
parser = CommandParser(None, usage="%(prog)s subcommand [options] [args]",
add_help=False)
parser.add_argument('--local_test_settings',
dest="local_test_settings")
options, args = parser.parse_known_args(argv)
if options.local_test_settings is None:
local_settings_file = "local_settings_example.py"
else:
local_settings_file = options.local_test_settings
if os.path.split(local_settings_file)[0] == "":
local_settings_file = os.path.join(
local_settings_dir, local_settings_file)
if os.path.abspath(local_settings_file) == os.path.abspath(
os.path.join(local_settings_dir, "local_settings.py")):
raise CommandError(
"Using production local_settings for tests is not "
"allowed due to security reason."
)
if not os.path.isfile(local_settings_file):
raise CommandError(
"file '%s' does not exist" % local_settings_file
)
return local_settings_file
def add_arguments(self, parser: CommandParser):
parser.add_argument('locations', nargs='+', type=str)
parser.add_argument('--clear',
action='store_true',
dest='clear',
help='Clear database')
parser.add_argument('--native',
action='store_true',
dest='native',
help='Store plugins as native')
def add_arguments(self, parser: CommandParser):
parser.description = self.help
parser.add_argument(
'command',
type=str,
help='One of: clone, pull, build, publish, update. Note: update = '
'pull + build + publish.')
parser.add_argument('repository',
type=str,
help='Local repo name (a simple directory name).')
parser.add_argument(
'--url',
type=str,
help='URL to clone from, e.g. Github. Wrap in quotes?')
def add_arguments(self, parser: CommandParser):
parser.add_argument("num_users",
type=int,
help="The number of total users to create")
parser.add_argument(
"num_active",
type=int,
help="The number of active users to create. Must be <= num_users",
)
parser.add_argument(
"num_applications",
type=int,
help="The number of applications to create. Must be <= num_active",
)
def add_arguments(self, parser: CommandParser) -> None:
class SubCommandParser(CommandParser):
def __init__(self, **kwargs: Any) -> None:
super().__init__(**kwargs)
subparsers = parser.add_subparsers(
dest='scmd', title="subcommands",
parser_class=SubCommandParser) # type: argparse._SubParsersAction
create_sp = subparsers.add_parser("create", help="Create user groups")
create_sp.add_argument("groupname",
help="The name of the group to create")
list_sp = subparsers.add_parser("list")
list_sp.add_argument("--format",
dest="format",
choices=["json", "table"],
default="table",
help="The output format for the results")
remove_sp = subparsers.add_parser("remove", help="Remove user group")
remove_sp.add_argument("groupname", help="The group's UUID or name")
def add_arguments(self, parser: CommandParser) -> None:
"""Add arguments related to the `wait_for_postgres` function."""
super().add_arguments(parser)
parser.add_argument(
"--no-superuser",
action="store_false",
dest="superuser",
help="Tells Django to NOT create a superuser account.",
)
parser.add_argument(
"--skip-migrations",
action="store_false",
dest="migrate",
help="Tells Django to SKIP running migrations.",
)
parser.add_argument(
"--no-collectstatic",
action="store_false",
dest="collectstatic",
help="Tells Django to SKIP running collectstatic.",
)
# It's nice if "./manage.py test" works out-of-the-box, so work out if
# "test" is the subcommand and the user hasn't specified a settings
# module with --settings. If so, use the settings module for
# non-country-specific tests that we know has the right INSTALLED_APPS
# and tests are expected to pass with. This won't help confusion if
# someone uses "django-admin.py test" instead, but it's some help...
# (Note that if someone's set the DJANGO_SETTINGS_MODULE environment
# variable, this default won't be used either.)
try:
subcommand = sys.argv[1]
except IndexError:
subcommand = 'help'
parser = CommandParser(None)
parser.add_argument('--settings')
try:
options, args = parser.parse_known_args(sys.argv[2:])
except:
# Ignore any errors at this point; the arguments will be parsed
# again shortly anyway.
args = []
run_default_tests = (subcommand == 'test' and not options.settings)
if __name__ == "__main__":
if run_default_tests:
settings_module = 'mysite.settings.tests'
else:
def __call__(self, argv=None):
if argv is None:
argv = sys.argv[1:]
warnings.filterwarnings('ignore',
module="IPython",
category=DeprecationWarning)
warnings.filterwarnings("ignore", module="distutils")
try:
warnings.filterwarnings("ignore", category=ResourceWarning)
except NameError:
pass
warnings.filterwarnings("ignore", "invalid escape sequence",
DeprecationWarning)
# Ignore a deprecation warning in distutils.spawn for python 3.4+
if sys.version_info > (3, 4):
warnings.filterwarnings("ignore", "the imp module", Warning)
warnings.filterwarnings('once', 'Selenium support for PhantomJS',
Warning)
import django
from django.core.management import CommandParser, handle_default_options
# Ignore a python 3.6 DeprecationWarning in ModelBase.__new__ that isn't
# fixed in Django 1.x
if sys.version_info > (3, 6) and django.VERSION < (2, ):
warnings.filterwarnings("ignore", "__class__ not set defining",
DeprecationWarning)
parser = CommandParser(None,
usage="%(prog)s [options] [args]",
add_help=False)
default_settings = os.environ.get('DJANGO_SETTINGS_MODULE',
self.default_settings_module)
parser.add_argument('--settings', default=default_settings)
parser.add_argument('--pythonpath')
parser.add_argument('--testrunner',
action='store',
default='django_admin_testutils.DiscoverRunner')
parser.add_argument('args', nargs='*')
options, remaining_args = parser.parse_known_args(argv)
handle_default_options(options)
test_labels = options.args or [self.default_test_label]
flags = ['--testrunner=%s' % options.testrunner] + remaining_args
from django.conf import settings
if 'grappelli' in settings.INSTALLED_APPS:
# Grappelli uses the deprecated django.conf.urls.patterns, but we
# don't want this to fail our tests.
warnings.filterwarnings("ignore", "django.conf.urls.patterns",
Warning)
self.execute(flags, test_labels)
def add_arguments(self, parser: CommandParser):
parser.add_argument('path', required=False, nargs='+', type=str, help='The import path.')
parser.add_argument('--from', nargs=1, type=str, help='The import source (fs|nemo). ')
def __call__(self, argv=None):
from django.conf import settings
from django.core.management import CommandParser, handle_default_options
if argv is None:
argv = sys.argv[1:]
try:
warnings.filterwarnings("ignore", category=ResourceWarning)
except NameError:
pass
warnings.filterwarnings('ignore',
module="IPython",
category=DeprecationWarning)
warnings.filterwarnings("ignore", module="distutils")
warnings.filterwarnings("ignore", "invalid escape sequence",
DeprecationWarning)
warnings.filterwarnings("ignore", "the imp module", Warning)
warnings.filterwarnings('once', 'Selenium support for PhantomJS',
Warning)
default_settings = os.environ.get('DJANGO_SETTINGS_MODULE',
self.default_settings_module)
parser = CommandParser(usage="%(prog)s [options] [args]",
add_help=False)
parser.add_argument('--settings', default=default_settings)
parser.add_argument('--pythonpath')
parser.add_argument('--testrunner',
action='store',
default='selenosis.DiscoverRunner')
parser.add_argument('args', nargs='*')
options, remaining_args = parser.parse_known_args(argv)
handle_default_options(options)
test_labels = options.args or [self.default_test_label]
flags = ['--testrunner=%s' % options.testrunner] + remaining_args
if 'grappelli' in settings.INSTALLED_APPS:
# Grappelli uses the deprecated django.conf.urls.patterns, but we
# don't want this to fail our tests.
warnings.filterwarnings("ignore", "django.conf.urls.patterns",
Warning)
self.execute(flags, test_labels)
def __prepare_parser(self):
"""
Create and return the ``ArgumentParser`` which will be used to
parse the arguments to this command.
"""
self._parser = CommandParser(
prog='%s' % getattr(self, "__crawler_name__"),
description="Crawler settings",
formatter_class=DjangoHelpFormatter,
missing_args_message=getattr(self, 'missing_args_message', None),
called_from_command_line=getattr(self, '_called_from_command_line',
None),
)
self._parser.add_argument('--version',
action='version',
version=self.__get_version())
self._parser.add_argument(
'-v',
'--verbosity',
action='store',
dest='verbosity',
default=1,
type=int,
choices=[0, 1, 2, 3],
help='Verbosity level; 0=minimal output, 1=normal output, '
'2=verbose output, 3=very verbose output',
)
self._parser.add_argument(
'--settings',
help=('The Python path to a settings module, e.g. '
'"myproject.settings.main". If this isn\'t provided, the '
'DJANGO_SETTINGS_MODULE environment variable will be used.'),
)
self._parser.add_argument(
'--pythonpath',
help='A directory to add to the Python path, e.g. '
'"/home/djangoprojects/myproject".',
)
self._parser.add_argument('--traceback',
action='store_true',
help='Raise on CommandError exceptions')
self._parser.add_argument(
'--no-color',
action='store_true',
dest='no_color',
help="Don't colorize the command output.",
)
self._parser.add_argument(
'--force-color',
action='store_true',
help='Force colorization of the command output.',
)
# Cache storage
# In production this folder could be volatile, losing all the data
# when the crawler finish. You can use the database or GCP to
# save permanently files or content
self._parser.add_argument(
'--cache-dir',
required=False,
action='store',
dest='cache_dir',
default="fs:///data/harvest/permanent",
type=str,
help="The path where we will leave the files."
" Ex. fs:///data/harvest/permanent"
" gs://davinci_harvest")
# Local path for manipulate files. This storage is volatile in nature.
self._parser.add_argument(
'--local-dir',
required=False,
action='store',
dest='local_dir',
default="fs///data/harvest/volatile",
type=str,
help="The path where we will leave the files."
" Ex. fs///data/harvest/volatile")
# Parallel Workers
self._parser.add_argument(
'--workers-num',
required=False,
action='store',
dest='workers_num',
default=10,
type=int,
help="The number of workers (threads) to launch in parallel")
# Location of the bin folder of the PhantomJS library
self._parser.add_argument(
'--phantomjs-path',
required=False,
action='store',
dest='phantomjs_path',
default=None,
type=str,
help="Absolute path to the bin directory of the PhantomJS library."
"Ex. '/phantomjs-2.1.1-macosx/bin/phantomjs'")
# Location of the bin folder of the PhantomJS library
self._parser.add_argument(
'--chromium-bin-file',
required=False,
action='store',
dest='chromium_bin_file',
default=None,
type=str,
help="Absolute path to the Chromium bin file."
"Ex. '/Applications/Chromium.app/Contents/MacOS/Chromium'")
# GCP Project
self._parser.add_argument(
'--io-gs-project',
required=False,
action='store',
dest='io_gs_project',
default=None,
type=str,
help="If we are using Google Storage to persist the files, we "
" could need to inform about the project of the bucket."
"Ex. centering-badge-212119")
# Current execution time
self._parser.add_argument(
'--current-execution-date',
required=False,
action='store',
dest='current_execution_date',
default=datetime.utcnow(),
type=mk_datetime,
help="The current time we are starting the crawler (UTC)"
" Ex. '2008-09-03T20:56:35.450686Z")
# Last execution time
self._parser.add_argument(
'--last-execution-date',
required=False,
action='store',
dest='last_execution_date',
default=None,
type=mk_datetime,
help="The last time we executed the crawler (UTC)"
" Ex. '2007-09-03T20:56:35.450686Z")
self.add_arguments(self._parser)
class Crawler(metaclass=ABCMeta):
# The unique name of the crawler to be identified in the system
__crawler_name__ = None
def __init__(self) -> None:
super().__init__()
if not hasattr(self, "__crawler_name__"):
raise RuntimeError(
"The crawler {} must specify "
"class Meta attribute '__crawler_name__'".format(
self.__class__))
self.__prepare_parser()
@classmethod
def get_web_driver(cls, **options):
"""
Initialized the Web Driver to allow dynamic web processing
:param options:
:return: the driver informed in the options (chromium has preference)
"""
driver = None
# Chromium folder
chromium_file = options.get("chromium_bin_file", None)
if chromium_file:
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument("--disable-features=NetworkService")
chrome_options.binary_location = chromium_file
driver = webdriver.Chrome(chrome_options=chrome_options)
_logger.info(
"Using CHROMIUM as Dynamic Web Driver. Driver {}".format(
repr(driver)))
else:
# PhantomJS folder
phantomjs_path = options.get("phantomjs_path", None)
if phantomjs_path:
driver = webdriver.PhantomJS(executable_path=phantomjs_path)
_logger.info(
"Using PHANTOMJS as Dynamic Web Driver. Driver {}".format(
repr(driver)))
if not driver:
_logger.warning("No Dynamic Web Driver loaded!!! "
"(no Chromium neither PhantomJS specified)")
return driver
def __get_version(self):
"""
Return the Django version, which should be correct for all built-in
Django commands. User-supplied commands can override this method to
return their own version.
"""
return django.get_version()
def __prepare_parser(self):
"""
Create and return the ``ArgumentParser`` which will be used to
parse the arguments to this command.
"""
self._parser = CommandParser(
prog='%s' % getattr(self, "__crawler_name__"),
description="Crawler settings",
formatter_class=DjangoHelpFormatter,
missing_args_message=getattr(self, 'missing_args_message', None),
called_from_command_line=getattr(self, '_called_from_command_line',
None),
)
self._parser.add_argument('--version',
action='version',
version=self.__get_version())
self._parser.add_argument(
'-v',
'--verbosity',
action='store',
dest='verbosity',
default=1,
type=int,
choices=[0, 1, 2, 3],
help='Verbosity level; 0=minimal output, 1=normal output, '
'2=verbose output, 3=very verbose output',
)
self._parser.add_argument(
'--settings',
help=('The Python path to a settings module, e.g. '
'"myproject.settings.main". If this isn\'t provided, the '
'DJANGO_SETTINGS_MODULE environment variable will be used.'),
)
self._parser.add_argument(
'--pythonpath',
help='A directory to add to the Python path, e.g. '
'"/home/djangoprojects/myproject".',
)
self._parser.add_argument('--traceback',
action='store_true',
help='Raise on CommandError exceptions')
self._parser.add_argument(
'--no-color',
action='store_true',
dest='no_color',
help="Don't colorize the command output.",
)
self._parser.add_argument(
'--force-color',
action='store_true',
help='Force colorization of the command output.',
)
# Cache storage
# In production this folder could be volatile, losing all the data
# when the crawler finish. You can use the database or GCP to
# save permanently files or content
self._parser.add_argument(
'--cache-dir',
required=False,
action='store',
dest='cache_dir',
default="fs:///data/harvest/permanent",
type=str,
help="The path where we will leave the files."
" Ex. fs:///data/harvest/permanent"
" gs://davinci_harvest")
# Local path for manipulate files. This storage is volatile in nature.
self._parser.add_argument(
'--local-dir',
required=False,
action='store',
dest='local_dir',
default="fs///data/harvest/volatile",
type=str,
help="The path where we will leave the files."
" Ex. fs///data/harvest/volatile")
# Parallel Workers
self._parser.add_argument(
'--workers-num',
required=False,
action='store',
dest='workers_num',
default=10,
type=int,
help="The number of workers (threads) to launch in parallel")
# Location of the bin folder of the PhantomJS library
self._parser.add_argument(
'--phantomjs-path',
required=False,
action='store',
dest='phantomjs_path',
default=None,
type=str,
help="Absolute path to the bin directory of the PhantomJS library."
"Ex. '/phantomjs-2.1.1-macosx/bin/phantomjs'")
# Location of the bin folder of the PhantomJS library
self._parser.add_argument(
'--chromium-bin-file',
required=False,
action='store',
dest='chromium_bin_file',
default=None,
type=str,
help="Absolute path to the Chromium bin file."
"Ex. '/Applications/Chromium.app/Contents/MacOS/Chromium'")
# GCP Project
self._parser.add_argument(
'--io-gs-project',
required=False,
action='store',
dest='io_gs_project',
default=None,
type=str,
help="If we are using Google Storage to persist the files, we "
" could need to inform about the project of the bucket."
"Ex. centering-badge-212119")
# Current execution time
self._parser.add_argument(
'--current-execution-date',
required=False,
action='store',
dest='current_execution_date',
default=datetime.utcnow(),
type=mk_datetime,
help="The current time we are starting the crawler (UTC)"
" Ex. '2008-09-03T20:56:35.450686Z")
# Last execution time
self._parser.add_argument(
'--last-execution-date',
required=False,
action='store',
dest='last_execution_date',
default=None,
type=mk_datetime,
help="The last time we executed the crawler (UTC)"
" Ex. '2007-09-03T20:56:35.450686Z")
self.add_arguments(self._parser)
def get_parser(self):
return self._parser
def add_arguments(self, parser):
pass
@abc.abstractmethod
def crawl_params(self, **options):
raise NotImplementedError()
@abc.abstractmethod
def crawl(self, crawling_params, options):
raise NotImplementedError()
def add_arguments(self, parser: CommandParser):
parser.add_argument(
'locks', type=str, nargs='+',
help='One or more lockmgr lock names (as positional args) to release the locks for'
)
