def coll6(): return rh.Collection('test', 'coll6', reference_fields='thing--test:coll5', index_fields='z')
def coll4(): return rh.Collection('test', 'coll4', index_fields='a, b, c')
def coll5(): return rh.Collection('test', 'coll5', unique_field='name', index_fields='status')
def coll2(): return rh.Collection('test', 'coll2', json_fields='data')
def coll3(): return rh.Collection('test', 'coll3', index_fields='a', json_fields='data')
'return': { 'fields': RETURN_FIELDS, 'prompt': 'Select return fields you are interested in (separate by space)' }, 'orderby': { 'fields': ORDER_BY_FIELDS, 'prompt': 'Select order by fields you are interested in (separate by space)' } } SELECTED_FIELDS = rh.Collection('jira', 'fields', index_fields='name,type', json_fields='selected', rx_type='(return|orderby)') SAVED_QUERIES = rh.Collection( 'jira', 'queries', index_fields='name', ) def get_session(): """Return an authenticated session object""" if not JIRA_URL or not JIRA_API_TOKEN or not JIRA_API_USER: raise Exception( '\nPlease define JIRA_URL, JIRA_API_TOKEN, and JIRA_API_USER environment vars..\n\n'
def coll1(): return rh.Collection('test', 'coll1')
import aws_info_helper as ah import input_helper as ih import dt_helper as dh from functools import partial try: import redis_helper as rh from redis import ConnectionError as RedisConnectionError except ImportError: AWS_EC2 = None else: try: AWS_EC2 = rh.Collection( 'aws', 'ec2', unique_field='id', index_fields= 'profile, type, pem, az, subnet, ami, name, status, sshuser', json_fields='sg', insert_ts=True) except RedisConnectionError: AWS_EC2 = None INSTANCE_FILTER_KEY_CONDITIONS = { 'Tags__Value': lambda x: x.get('Key') == 'Name' } INSTANCE_KEY_VALUE_CASTING = { 'LaunchTime': lambda x: dh.utc_float_to_pretty(dh.dt_to_float_string(x)) } INSTANCE_KEY_NAME_MAPPING = {
try: import redis_helper as rh import input_helper as ih from redis import ConnectionError as RedisConnectionError except ImportError: COMMENTS = None else: try: COMMENTS = rh.Collection( 'av', 'comment', index_fields='basename', json_fields=','.join(ih.SPECIAL_TEXT_RETURN_FIELDS), insert_ts=True, ) except RedisConnectionError: COMMENTS = None
import input_helper as ih import dt_helper as dh from functools import partial from os.path import isdir, dirname, basename, join from os import makedirs try: import redis_helper as rh from redis import ConnectionError as RedisConnectionError except ImportError: AWS_S3 = None AWS_S3_LAST_FILE = None else: try: AWS_S3 = rh.Collection( 'aws', 's3', index_fields='profile, bucket', ) AWS_S3_LAST_FILE = rh.Collection( 'aws', 's3_last', index_fields='profile, bucket, prefix', ) except RedisConnectionError: AWS_S3 = None AWS_S3_LAST_FILE = None BUCKET_KEY_VALUE_CASTING = { 'CreationDate': lambda x: dh.utc_float_to_pretty(dh.dt_to_float_string(x)) }
# Demo: https://asciinema.org/a/75kl95ty9vg2jl93pfz9fbs9q?t=1:00 import redis_helper as rh from pprint import pprint urls = rh.Collection('web', 'url', unique_field='name', index_fields='domain, _type') def add_urls(): urls.add( name='mini-16.04.bash', url= 'https://raw.githubusercontent.com/kenjyco/x200/master/sysinstall/apt-get/mini-16.04.bash', domain='githubusercontent.com', subdomain='raw', _type='setup_script', ) urls.add( name='x200 sysinstall readme', url= 'https://github.com/kenjyco/x200/blob/master/sysinstall/apt-get/README.md', domain='github.com', _type='markdown', ) urls.add( name='dotfiles Background.md', url='https://github.com/kenjyco/dotfiles/blob/master/Background.md', domain='github.com',
def __init__(self, *args, **kwargs): """ kwargs: - chfunc_dict: an OrderedDict where keys are characters and values are 2-item tuples - first item is a function accepting no arguments - use `functools.partial` if needed - second item is some 'help text' - prompt: string to display when asking for input (default is '\n> ') - name: value of the 'name' argument for `redis_helper.Collection` - break_chars: list of characters that can be used to break the loop - if any char is used in chfunc_dict, the associated function will be called before breaking the input loop - input_hook: a callable (that receives `**kwargs`) to do extra things with user input received after '-' is pressed - the dict returned from the `input_helper.user_input_fancy` func will be used as kwargs to the `input_hook` - pre_input_hook: a callable (receiving no args) that returns a dict of info (as soon as '-' is pressed) that will be passed to `input_hook` - post_input_hook: a callable (receiving no args) that returns a dict of info (as soon as user_input_fancy completes) that will be passed to `input_hook` """ self._chfunc_dict = kwargs.pop('chfunc_dict', OrderedDict()) self._prompt = kwargs.pop('prompt', '\n> ') self._loop_name = kwargs.pop('name', 'default') self._break_chars = kwargs.pop('break_chars', []) self._input_hook = kwargs.pop('input_hook', None) self._pre_input_hook = kwargs.pop('pre_input_hook', None) self._post_input_hook = kwargs.pop('post_input_hook', None) self._DONT_LOG_CMDS = [ 'docstrings', 'shortcuts', 'errors', 'history', ] self._collection = rh.Collection( 'chloop-log', self._loop_name, index_fields='cmd,status,error_type', json_fields='args,value' ) self._wishlist = rh.Collection( 'chloop-wish', self._loop_name, index_fields='ch,cmd' ) self._method_names = [ m[0] for m in inspect.getmembers(self) if not m[0].startswith('_') ] self._method_docs = { method: getattr(self, method).__doc__ or '' for method in self._method_names } if type(self._chfunc_dict) != OrderedDict: self._chfunc_dict = OrderedDict(sorted( self._chfunc_dict.items(), key=lambda k: k[1][1] ))
# Demo: https://asciinema.org/a/101422?t=1:10 import redis_helper as rh import random import time from pprint import pprint request_logs = rh.Collection( 'log', 'request', index_fields='status, uri, host', json_fields='request, response, headers' ) def generate_request_log(add=True, show=False): """Generate a request_log dict and return 2-item tuple (_id, request_log) - add: if True, automatically add the genarated request_log to 'request_logs' - show: if True, pprint the generated data to the screen """ request_log = { 'method': random.choice(['get', 'post']), 'status': random.choice([200] * 1500 + [400]), 'host': random.choice(['dogs.com', 'cats.com', 'birds.com', 'tigers.com']), 'uri': random.choice(['/colors', '/shapes', '/breeds', '/search']), } if request_log['method'] == 'get': request_log['querystring'] = '?x={}&y={}'.format(
import redis_helper as rh import input_helper as ih from redis import ConnectionError as RedisConnectionError except ImportError: QUERIES = None URLS = None FILES = None AUDIO_COMMENTS = None VIDEO_COMMENTS = None else: try: QUERIES = rh.Collection( 'av', 'query', unique_field='query', json_fields='basenames,related', insert_ts=True, ) URLS = rh.Collection( 'av', 'url', unique_field='url', index_fields='domain', json_fields='basenames', insert_ts=True, ) FILES = rh.Collection( 'av',
import aws_info_helper as ah import input_helper as ih import dt_helper as dh from functools import partial try: import redis_helper as rh from redis import ConnectionError as RedisConnectionError except ImportError: AWS_ROUTE53 = None else: try: AWS_ROUTE53 = rh.Collection( 'aws', 'route53', index_fields='profile, domain, subdomain, type, external', rx_external='(yes|no)', ) except RedisConnectionError: AWS_ROUTE53 = None ZONE_KEY_VALUE_CASTING = { 'Name': lambda x: x.strip('.') } RESOURCE_KEY_VALUE_CASTING = { 'Name': lambda x: x.strip('.'), 'ResourceRecords__Value': lambda x: x.strip('.') } ZONE_KEY_NAME_MAPPING = {
import redis_helper as rh sample = rh.Collection('ns', 'sample', unique_field='name', index_fields='status', json_fields='data', rx_name='\S{4,6}', rx_status='(active|inactive|cancelled)', rx_aws='[a-z]+\-[0-9a-f]+', insert_ts=True) uses_sample = rh.Collection('ns', 'uses_sample', index_fields='z', rx_thing='\S{4,6}', reference_fields='thing--ns:sample')
logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) file_handler = logging.FileHandler(LOGFILE, mode='a') file_handler.setLevel(logging.DEBUG) file_handler.setFormatter( logging.Formatter( '%(asctime)s - %(levelname)s - %(funcName)s: %(message)s')) console_handler = logging.StreamHandler() console_handler.setLevel(logging.INFO) console_handler.setFormatter(logging.Formatter('%(asctime)s: %(message)s')) logger.addHandler(file_handler) logger.addHandler(console_handler) PLAYERS = rh.Collection('mmo', 'player', unique_field='handle', index_fields='email, status', insert_ts=True) CHARACTERS = rh.Collection( 'mmo', 'character', unique_field='name', index_fields='species, specialty, rank, player, status', insert_ts=True) ITEMS = rh.Collection( 'mmo', 'item', unique_field='name', )
import os.path import boto3 import settings_helper as sh import bg_helper as bh from os import walk from botocore.exceptions import EndpointConnectionError, ClientError, ProfileNotFound try: import redis_helper as rh from redis import ConnectionError as RedisConnectionError except ImportError: AWS_IP = None else: try: AWS_IP = rh.Collection( 'aws', 'ip', index_fields='profile, ip, name, source, instance', reference_fields='instance--aws:ec2', insert_ts=True) except RedisConnectionError: AWS_IP = None get_setting = sh.settings_getter(__name__) EC2_INSTANCE_KEYS = get_setting('EC2_INSTANCE_KEYS') EC2_INSTANCE_INFO_FORMAT = get_setting('EC2_INSTANCE_INFO_FORMAT') EC2_ADDRESS_KEYS = get_setting('EC2_ADDRESS_KEYS') ROUTE53_ZONE_KEYS = get_setting('ROUTE53_ZONE_KEYS') ROUTE53_RESOURCE_KEYS = get_setting('ROUTE53_RESOURCE_KEYS') ROUTE53_RESOURCE_INFO_FORMAT = get_setting('ROUTE53_RESOURCE_INFO_FORMAT') IP_RX = re.compile(r'(?:\d{1,3}\.)+\d{1,3}') SSH_FAILED_OUTPUT_RX = re.compile( r'.*(Timeout|Permission denied|Connection closed by|Connection timed out).*',