def coll6():
return rh.Collection('test',
'coll6',
reference_fields='thing--test:coll5',
index_fields='z')
def coll4():
return rh.Collection('test', 'coll4', index_fields='a, b, c')
def coll5():
return rh.Collection('test',
'coll5',
unique_field='name',
index_fields='status')
def coll2():
return rh.Collection('test', 'coll2', json_fields='data')
def coll3():
return rh.Collection('test', 'coll3', index_fields='a', json_fields='data')
'return': {
'fields': RETURN_FIELDS,
'prompt':
'Select return fields you are interested in (separate by space)'
},
'orderby': {
'fields':
ORDER_BY_FIELDS,
'prompt':
'Select order by fields you are interested in (separate by space)'
}
}
SELECTED_FIELDS = rh.Collection('jira',
'fields',
index_fields='name,type',
json_fields='selected',
rx_type='(return|orderby)')
SAVED_QUERIES = rh.Collection(
'jira',
'queries',
index_fields='name',
)
def get_session():
"""Return an authenticated session object"""
if not JIRA_URL or not JIRA_API_TOKEN or not JIRA_API_USER:
raise Exception(
'\nPlease define JIRA_URL, JIRA_API_TOKEN, and JIRA_API_USER environment vars..\n\n'
def coll1():
return rh.Collection('test', 'coll1')
import aws_info_helper as ah
import input_helper as ih
import dt_helper as dh
from functools import partial
try:
import redis_helper as rh
from redis import ConnectionError as RedisConnectionError
except ImportError:
AWS_EC2 = None
else:
try:
AWS_EC2 = rh.Collection(
'aws',
'ec2',
unique_field='id',
index_fields=
'profile, type, pem, az, subnet, ami, name, status, sshuser',
json_fields='sg',
insert_ts=True)
except RedisConnectionError:
AWS_EC2 = None
INSTANCE_FILTER_KEY_CONDITIONS = {
'Tags__Value': lambda x: x.get('Key') == 'Name'
}
INSTANCE_KEY_VALUE_CASTING = {
'LaunchTime': lambda x: dh.utc_float_to_pretty(dh.dt_to_float_string(x))
}
INSTANCE_KEY_NAME_MAPPING = {
try:
import redis_helper as rh
import input_helper as ih
from redis import ConnectionError as RedisConnectionError
except ImportError:
COMMENTS = None
else:
try:
COMMENTS = rh.Collection(
'av',
'comment',
index_fields='basename',
json_fields=','.join(ih.SPECIAL_TEXT_RETURN_FIELDS),
insert_ts=True,
)
except RedisConnectionError:
COMMENTS = None
import input_helper as ih
import dt_helper as dh
from functools import partial
from os.path import isdir, dirname, basename, join
from os import makedirs
try:
import redis_helper as rh
from redis import ConnectionError as RedisConnectionError
except ImportError:
AWS_S3 = None
AWS_S3_LAST_FILE = None
else:
try:
AWS_S3 = rh.Collection(
'aws',
's3',
index_fields='profile, bucket',
)
AWS_S3_LAST_FILE = rh.Collection(
'aws',
's3_last',
index_fields='profile, bucket, prefix',
)
except RedisConnectionError:
AWS_S3 = None
AWS_S3_LAST_FILE = None
BUCKET_KEY_VALUE_CASTING = {
'CreationDate': lambda x: dh.utc_float_to_pretty(dh.dt_to_float_string(x))
}
# Demo: https://asciinema.org/a/75kl95ty9vg2jl93pfz9fbs9q?t=1:00
import redis_helper as rh
from pprint import pprint
urls = rh.Collection('web',
'url',
unique_field='name',
index_fields='domain, _type')
def add_urls():
urls.add(
name='mini-16.04.bash',
url=
'https://raw.githubusercontent.com/kenjyco/x200/master/sysinstall/apt-get/mini-16.04.bash',
domain='githubusercontent.com',
subdomain='raw',
_type='setup_script',
)
urls.add(
name='x200 sysinstall readme',
url=
'https://github.com/kenjyco/x200/blob/master/sysinstall/apt-get/README.md',
domain='github.com',
_type='markdown',
)
urls.add(
name='dotfiles Background.md',
url='https://github.com/kenjyco/dotfiles/blob/master/Background.md',
domain='github.com',
def __init__(self, *args, **kwargs):
"""
kwargs:
- chfunc_dict: an OrderedDict where keys are characters and values are
2-item tuples
- first item is a function accepting no arguments
- use `functools.partial` if needed
- second item is some 'help text'
- prompt: string to display when asking for input (default is '\n> ')
- name: value of the 'name' argument for `redis_helper.Collection`
- break_chars: list of characters that can be used to break the loop
- if any char is used in chfunc_dict, the associated function will
be called before breaking the input loop
- input_hook: a callable (that receives `**kwargs`) to do extra things
with user input received after '-' is pressed
- the dict returned from the `input_helper.user_input_fancy` func
will be used as kwargs to the `input_hook`
- pre_input_hook: a callable (receiving no args) that returns a dict of
info (as soon as '-' is pressed) that will be passed to `input_hook`
- post_input_hook: a callable (receiving no args) that returns a dict of
info (as soon as user_input_fancy completes) that will be passed to
`input_hook`
"""
self._chfunc_dict = kwargs.pop('chfunc_dict', OrderedDict())
self._prompt = kwargs.pop('prompt', '\n> ')
self._loop_name = kwargs.pop('name', 'default')
self._break_chars = kwargs.pop('break_chars', [])
self._input_hook = kwargs.pop('input_hook', None)
self._pre_input_hook = kwargs.pop('pre_input_hook', None)
self._post_input_hook = kwargs.pop('post_input_hook', None)
self._DONT_LOG_CMDS = [
'docstrings', 'shortcuts', 'errors', 'history',
]
self._collection = rh.Collection(
'chloop-log',
self._loop_name,
index_fields='cmd,status,error_type',
json_fields='args,value'
)
self._wishlist = rh.Collection(
'chloop-wish',
self._loop_name,
index_fields='ch,cmd'
)
self._method_names = [
m[0]
for m in inspect.getmembers(self) if not m[0].startswith('_')
]
self._method_docs = {
method: getattr(self, method).__doc__ or ''
for method in self._method_names
}
if type(self._chfunc_dict) != OrderedDict:
self._chfunc_dict = OrderedDict(sorted(
self._chfunc_dict.items(),
key=lambda k: k[1][1]
))
# Demo: https://asciinema.org/a/101422?t=1:10
import redis_helper as rh
import random
import time
from pprint import pprint
request_logs = rh.Collection(
'log',
'request',
index_fields='status, uri, host',
json_fields='request, response, headers'
)
def generate_request_log(add=True, show=False):
"""Generate a request_log dict and return 2-item tuple (_id, request_log)
- add: if True, automatically add the genarated request_log to 'request_logs'
- show: if True, pprint the generated data to the screen
"""
request_log = {
'method': random.choice(['get', 'post']),
'status': random.choice([200] * 1500 + [400]),
'host': random.choice(['dogs.com', 'cats.com', 'birds.com', 'tigers.com']),
'uri': random.choice(['/colors', '/shapes', '/breeds', '/search']),
}
if request_log['method'] == 'get':
request_log['querystring'] = '?x={}&y={}'.format(
import redis_helper as rh
import input_helper as ih
from redis import ConnectionError as RedisConnectionError
except ImportError:
QUERIES = None
URLS = None
FILES = None
AUDIO_COMMENTS = None
VIDEO_COMMENTS = None
else:
try:
QUERIES = rh.Collection(
'av',
'query',
unique_field='query',
json_fields='basenames,related',
insert_ts=True,
)
URLS = rh.Collection(
'av',
'url',
unique_field='url',
index_fields='domain',
json_fields='basenames',
insert_ts=True,
)
FILES = rh.Collection(
'av',
import aws_info_helper as ah
import input_helper as ih
import dt_helper as dh
from functools import partial
try:
import redis_helper as rh
from redis import ConnectionError as RedisConnectionError
except ImportError:
AWS_ROUTE53 = None
else:
try:
AWS_ROUTE53 = rh.Collection(
'aws',
'route53',
index_fields='profile, domain, subdomain, type, external',
rx_external='(yes|no)',
)
except RedisConnectionError:
AWS_ROUTE53 = None
ZONE_KEY_VALUE_CASTING = {
'Name': lambda x: x.strip('.')
}
RESOURCE_KEY_VALUE_CASTING = {
'Name': lambda x: x.strip('.'),
'ResourceRecords__Value': lambda x: x.strip('.')
}
ZONE_KEY_NAME_MAPPING = {
import redis_helper as rh
sample = rh.Collection('ns',
'sample',
unique_field='name',
index_fields='status',
json_fields='data',
rx_name='\S{4,6}',
rx_status='(active|inactive|cancelled)',
rx_aws='[a-z]+\-[0-9a-f]+',
insert_ts=True)
uses_sample = rh.Collection('ns',
'uses_sample',
index_fields='z',
rx_thing='\S{4,6}',
reference_fields='thing--ns:sample')
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
file_handler = logging.FileHandler(LOGFILE, mode='a')
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(
logging.Formatter(
'%(asctime)s - %(levelname)s - %(funcName)s: %(message)s'))
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.INFO)
console_handler.setFormatter(logging.Formatter('%(asctime)s: %(message)s'))
logger.addHandler(file_handler)
logger.addHandler(console_handler)
PLAYERS = rh.Collection('mmo',
'player',
unique_field='handle',
index_fields='email, status',
insert_ts=True)
CHARACTERS = rh.Collection(
'mmo',
'character',
unique_field='name',
index_fields='species, specialty, rank, player, status',
insert_ts=True)
ITEMS = rh.Collection(
'mmo',
'item',
unique_field='name',
)
import os.path
import boto3
import settings_helper as sh
import bg_helper as bh
from os import walk
from botocore.exceptions import EndpointConnectionError, ClientError, ProfileNotFound
try:
import redis_helper as rh
from redis import ConnectionError as RedisConnectionError
except ImportError:
AWS_IP = None
else:
try:
AWS_IP = rh.Collection(
'aws',
'ip',
index_fields='profile, ip, name, source, instance',
reference_fields='instance--aws:ec2',
insert_ts=True)
except RedisConnectionError:
AWS_IP = None
get_setting = sh.settings_getter(__name__)
EC2_INSTANCE_KEYS = get_setting('EC2_INSTANCE_KEYS')
EC2_INSTANCE_INFO_FORMAT = get_setting('EC2_INSTANCE_INFO_FORMAT')
EC2_ADDRESS_KEYS = get_setting('EC2_ADDRESS_KEYS')
ROUTE53_ZONE_KEYS = get_setting('ROUTE53_ZONE_KEYS')
ROUTE53_RESOURCE_KEYS = get_setting('ROUTE53_RESOURCE_KEYS')
ROUTE53_RESOURCE_INFO_FORMAT = get_setting('ROUTE53_RESOURCE_INFO_FORMAT')
IP_RX = re.compile(r'(?:\d{1,3}\.)+\d{1,3}')
SSH_FAILED_OUTPUT_RX = re.compile(
r'.*(Timeout|Permission denied|Connection closed by|Connection timed out).*',
