def _parse(self, params: List[str]) -> None: ''' todo: docs ''' try: logger().setLevel(getLevelName(params[0])) except: exit('Invalid log level {}'.format(params[0])) instance.config.set('logs', 'elvl', params[0]) logger().info('Set log level to %s', params[0])
def _parse(self, params: List[str]) -> None: ''' todo: docs ''' try: instance.config.read_string(params[0]) except: exit('Invalid config string {}'.format(params[0])) logger().info('Read config string %s', params[0])
def __init__(self, params: List[str]) -> None: ''' todo: docs ''' args = [] logger().debug('Parsing cli param %s', params.pop(0)[2:]) while params and not params[0].startswith('--'): args += [params.pop(0)] logger().debug('Passing cli args %s', str(args)) self._parse(args)
def pdf_to_pages(filename): # Load your PDF with open(filename, "rb") as f: try: logger().info('Reading %s', filename) pdf = pdftotext.PDF(f) # Iterate over all the pages for page in pdf: if page: yield page except: logger().error('Could not read %s', filename)
def main(self) -> None: try: instance.config = ConfigParser() instance.config.read_dict(defaultconfig()) logger().info('Started eisp with pid %s', getpid()) for i in [i for i in argv if i.startswith('--')]: try: mod('eisp.param.{}'.format(i[2:])).__dict__[i[2:]](argv) except: exit('Invalid parameter or argument to {}'.format(i[2:])) conf = dotdict(instance.config['data']) connections.create_connection(hosts=[conf.host]) delete_index(conf.index_name) create_index(conf.elastic_mapping, conf.index_name) for ok, info in helpers.parallel_bulk(connections.get_connection(), actions=index_pdfs( conf.index_name, conf.root), request_timeout=60, chunk_size=100, thread_count=8, queue_size=8): if not ok: print(info) except KeyboardInterrupt: print('\N{bomb}') except Exception as exception: logger().exception(exception) except SystemExit as exception: logger().critical(str(exception))
def _parse(self, params: List[str]) -> None: ''' todo: docs ''' instance.config.set('data', 'elastic_mapping', str(strtobool(params[0]))) logger().info('Set data elastic_mapping to %r', bool(strtobool(params[0])))
def _parse(self, params: List[str]) -> None: ''' todo: docs ''' instance.config.set('data', 'root', params[0]) logger().info('Set data root to %s', params[0])
def _parse(self, params: List[str]) -> None: ''' todo: docs ''' instance.config.set('data', 'index_name', str(strtobool(params[0]))) logger().info('Set data index_name to %r', bool(strtobool(params[0])))
def delete_index(index_name): idxs = connections.get_connection().indices logger().info('Dropping index %s', index_name) idxs.delete(ignore=404, index=index_name)
def create_index(path_to_elastic_mapping, index_name): idxs = connections.get_connection().indices logger().info('Creating index %s', index_name) idxs.create(index=index_name, body=load_elastic_mapping(path_to_elastic_mapping))