def cli(ctx, outdir, roster):
"""Scrapes rosters for Spiders using info in inmates.csv"""
all_links = build_dict(
format_records(
produce_records('inmates.csv'),
('IL County', lambda cell: cell.replace(' County', '').replace('. ', '').lower()),
('Roster Link', lambda cell: cell)
)
)
roster_urls = dict((path.stem, all_links[path.stem]) for path in spider_paths)
if outdir:
outdirpath = Path(outdir)
outdirpath.mkdir(exist_ok=True)
else:
outdirpath = None
if roster:
outopt = f'-o {outdirpath}/{roster}.json -s LOG_ENABLED=False' if outdirpath else ''
run_proc(split_args(f'{python} -m scrapy crawl -a domain={roster_urls[roster]} {roster} {outopt}'))
print('✅', roster)
else:
for roster, url in roster_urls.items():
outopt = f'-o {outdirpath}/{roster}.json -s LOG_ENABLED=False' if outdirpath else ''
run_proc(split_args(f'{python} -m scrapy crawl -a domain={url} {roster} {outopt}'))
print('✅', roster)
def do_add(self, args):
"""!NAME
add - Add a domain/record
!SYNOPSIS
add domain [domain option]
add domain host type value [mx] [ttl]
!DESCRIPTION
Add either a domain or a host record to HurricaneDNS. If the number of
arguments is 1 or 2, it will add the domain in arg 1, and then use arg
2 as option. If the number of arguments is between 4 and 6, it will
add the host record with optional MX and TTL.
"""
args = split_args(args)
try:
if len(args) and len(args) < 3:
domain = args[0]
extra = {}
if len(args) == 2:
(option, value) = args[1].split("=", 1)
extra[option] = value
self._get_hdns().add_domain(domain, **extra)
elif 6 >= len(args) >= 4:
self._get_hdns().add_record(*args)
else:
self._do_error('Invalid arguments')
except HurricaneDNS.HurricaneError as e:
self._do_error(e)
def do_del(self, args):
"""!NAME
del - Delete domain or host records
!SYNOPSIS
del domain
del domain host [type] [value] [mx] [ttl]
!DESCRIPTION
Delete either domain or host records. If only 1 argument is given, it
will use that as the domain name ot delete. Otherwise it will delete
host records that match the arguments.
"""
args = split_args(args)
try:
if len(args) == 1:
self._get_hdns().del_domain(args[0])
elif len(args) > 1:
self._get_hdns().del_records(*args)
except HurricaneDNS.HurricaneError as e:
self._do_error(e)
def do_ls(self, args):
"""!NAME
ls - List domains or host records
!SYNOPSIS
ls
ls domain [domain...]
!DESCRIPTION
Listing all the domains when there is no argument. Otherwise list all
host records from the specified domains.
"""
if args:
existing = self._get_hdns().cache_domains()
existing = set([item['domain'] for item in existing])
records = []
for domain in split_args(args):
if domain.lower() not in existing:
self._do_error('Invalid domain: ' + domain)
continue
records.extend(self._get_hdns().cache_records(domain))
if records:
maxhost = max([len(item['host']) for item in records])
maxvalue = max([len(item['value']) for item in records])
maxttl = max([len(item['ttl']) for item in records])
template = '%%%ds %%-5s %%-%ds %%%ds %%6s' % (maxhost, maxvalue, maxttl)
print template % ('HOST', 'TYPE', 'VALUE', 'TTL', 'MX')
for record in records:
print template % (record['host'], record['type'], record['value'], record['ttl'], record['mx'])
else:
domains = self._get_hdns().cache_domains()
domains.sort(key=lambda item: item['domain'])
print 'TYPE DOMAIN'
for domain in domains:
print '%-9s %s' % (domain['type'], domain['domain'])
def _split_args(self):
return split_args(self.args, posix=False)
def _split_args(self):
return split_args(self.args)
def _build_non_default_lang_docs(self, lang):
call(split_args('sphinx-build -b gettext . {:}'.format(self.SPHINX_DIRS['build']['locale'])))
call(split_args('sphinx-intl update -p {:} -l {:}'.format(self.SPHINX_DIRS['build']['locale'], lang)))
call(split_args('sphinx-intl build'))
call(split_args('sphinx-build -b html -d {:} -D language=\'{:}\' . {:}'.format(
self.SPHINX_DIRS['build']['doctrees'], lang, self.SPHINX_DIRS['build']['html'])))
def _build_default_lang_docs(self):
call(split_args('sphinx-build -b html -d {:} . {:}'.format(
self.SPHINX_DIRS['build']['doctrees'], self.SPHINX_DIRS['build']['html'])))
