def cli(ctx, outdir, roster): """Scrapes rosters for Spiders using info in inmates.csv""" all_links = build_dict( format_records( produce_records('inmates.csv'), ('IL County', lambda cell: cell.replace(' County', '').replace('. ', '').lower()), ('Roster Link', lambda cell: cell) ) ) roster_urls = dict((path.stem, all_links[path.stem]) for path in spider_paths) if outdir: outdirpath = Path(outdir) outdirpath.mkdir(exist_ok=True) else: outdirpath = None if roster: outopt = f'-o {outdirpath}/{roster}.json -s LOG_ENABLED=False' if outdirpath else '' run_proc(split_args(f'{python} -m scrapy crawl -a domain={roster_urls[roster]} {roster} {outopt}')) print('✅', roster) else: for roster, url in roster_urls.items(): outopt = f'-o {outdirpath}/{roster}.json -s LOG_ENABLED=False' if outdirpath else '' run_proc(split_args(f'{python} -m scrapy crawl -a domain={url} {roster} {outopt}')) print('✅', roster)
def do_add(self, args): """!NAME add - Add a domain/record !SYNOPSIS add domain [domain option] add domain host type value [mx] [ttl] !DESCRIPTION Add either a domain or a host record to HurricaneDNS. If the number of arguments is 1 or 2, it will add the domain in arg 1, and then use arg 2 as option. If the number of arguments is between 4 and 6, it will add the host record with optional MX and TTL. """ args = split_args(args) try: if len(args) and len(args) < 3: domain = args[0] extra = {} if len(args) == 2: (option, value) = args[1].split("=", 1) extra[option] = value self._get_hdns().add_domain(domain, **extra) elif 6 >= len(args) >= 4: self._get_hdns().add_record(*args) else: self._do_error('Invalid arguments') except HurricaneDNS.HurricaneError as e: self._do_error(e)
def do_del(self, args): """!NAME del - Delete domain or host records !SYNOPSIS del domain del domain host [type] [value] [mx] [ttl] !DESCRIPTION Delete either domain or host records. If only 1 argument is given, it will use that as the domain name ot delete. Otherwise it will delete host records that match the arguments. """ args = split_args(args) try: if len(args) == 1: self._get_hdns().del_domain(args[0]) elif len(args) > 1: self._get_hdns().del_records(*args) except HurricaneDNS.HurricaneError as e: self._do_error(e)
def do_ls(self, args): """!NAME ls - List domains or host records !SYNOPSIS ls ls domain [domain...] !DESCRIPTION Listing all the domains when there is no argument. Otherwise list all host records from the specified domains. """ if args: existing = self._get_hdns().cache_domains() existing = set([item['domain'] for item in existing]) records = [] for domain in split_args(args): if domain.lower() not in existing: self._do_error('Invalid domain: ' + domain) continue records.extend(self._get_hdns().cache_records(domain)) if records: maxhost = max([len(item['host']) for item in records]) maxvalue = max([len(item['value']) for item in records]) maxttl = max([len(item['ttl']) for item in records]) template = '%%%ds %%-5s %%-%ds %%%ds %%6s' % (maxhost, maxvalue, maxttl) print template % ('HOST', 'TYPE', 'VALUE', 'TTL', 'MX') for record in records: print template % (record['host'], record['type'], record['value'], record['ttl'], record['mx']) else: domains = self._get_hdns().cache_domains() domains.sort(key=lambda item: item['domain']) print 'TYPE DOMAIN' for domain in domains: print '%-9s %s' % (domain['type'], domain['domain'])
def _split_args(self): return split_args(self.args, posix=False)
def _split_args(self): return split_args(self.args)
def _build_non_default_lang_docs(self, lang): call(split_args('sphinx-build -b gettext . {:}'.format(self.SPHINX_DIRS['build']['locale']))) call(split_args('sphinx-intl update -p {:} -l {:}'.format(self.SPHINX_DIRS['build']['locale'], lang))) call(split_args('sphinx-intl build')) call(split_args('sphinx-build -b html -d {:} -D language=\'{:}\' . {:}'.format( self.SPHINX_DIRS['build']['doctrees'], lang, self.SPHINX_DIRS['build']['html'])))
def _build_default_lang_docs(self): call(split_args('sphinx-build -b html -d {:} . {:}'.format( self.SPHINX_DIRS['build']['doctrees'], self.SPHINX_DIRS['build']['html'])))