def _parse(self, params: List[str]) -> None:
'''
todo: docs
'''
try: logger().setLevel(getLevelName(params[0]))
except: exit('Invalid log level {}'.format(params[0]))
instance.config.set('logs', 'elvl', params[0])
logger().info('Set log level to %s', params[0])
def _parse(self, params: List[str]) -> None:
'''
todo: docs
'''
try:
instance.config.read_string(params[0])
except:
exit('Invalid config string {}'.format(params[0]))
logger().info('Read config string %s', params[0])
def __init__(self, params: List[str]) -> None:
'''
todo: docs
'''
args = []
logger().debug('Parsing cli param %s', params.pop(0)[2:])
while params and not params[0].startswith('--'):
args += [params.pop(0)]
logger().debug('Passing cli args %s', str(args))
self._parse(args)
def pdf_to_pages(filename):
# Load your PDF
with open(filename, "rb") as f:
try:
logger().info('Reading %s', filename)
pdf = pdftotext.PDF(f)
# Iterate over all the pages
for page in pdf:
if page:
yield page
except:
logger().error('Could not read %s', filename)
def main(self) -> None:
try:
instance.config = ConfigParser()
instance.config.read_dict(defaultconfig())
logger().info('Started eisp with pid %s', getpid())
for i in [i for i in argv if i.startswith('--')]:
try:
mod('eisp.param.{}'.format(i[2:])).__dict__[i[2:]](argv)
except:
exit('Invalid parameter or argument to {}'.format(i[2:]))
conf = dotdict(instance.config['data'])
connections.create_connection(hosts=[conf.host])
delete_index(conf.index_name)
create_index(conf.elastic_mapping, conf.index_name)
for ok, info in helpers.parallel_bulk(connections.get_connection(),
actions=index_pdfs(
conf.index_name,
conf.root),
request_timeout=60,
chunk_size=100,
thread_count=8,
queue_size=8):
if not ok:
print(info)
except KeyboardInterrupt:
print('\N{bomb}')
except Exception as exception:
logger().exception(exception)
except SystemExit as exception:
logger().critical(str(exception))
def _parse(self, params: List[str]) -> None:
'''
todo: docs
'''
instance.config.set('data', 'elastic_mapping', str(strtobool(params[0])))
logger().info('Set data elastic_mapping to %r', bool(strtobool(params[0])))
def _parse(self, params: List[str]) -> None:
'''
todo: docs
'''
instance.config.set('data', 'root', params[0])
logger().info('Set data root to %s', params[0])
def _parse(self, params: List[str]) -> None:
'''
todo: docs
'''
instance.config.set('data', 'index_name', str(strtobool(params[0])))
logger().info('Set data index_name to %r', bool(strtobool(params[0])))
def delete_index(index_name):
idxs = connections.get_connection().indices
logger().info('Dropping index %s', index_name)
idxs.delete(ignore=404, index=index_name)
def create_index(path_to_elastic_mapping, index_name):
idxs = connections.get_connection().indices
logger().info('Creating index %s', index_name)
idxs.create(index=index_name, body=load_elastic_mapping(path_to_elastic_mapping))