"""
S3 methods for PAWS library
"""
import boto3
from sensible.loginit import logger
log = logger("Paws")
def boto_s3_resource():
"""Create boto s3 resource"""
return boto3.resource("s3")
def download(bucket, key, filename, resource=None):
"""Downloads file from S3"""
if resource is None:
resource = boto_s3_resource()
log_msg = "Attempting to download {}, {}, {}".format(bucket, key, filename)
log.info(log_msg)
resource.meta.client.download_file(bucket, key, filename)
return filename
"""
Commandline Tool For Doing CSV operations:
* Aggregation
* TBD
"""
import sys
import click
from sensible.loginit import logger
import nlib
from nlib import csvops
from nlib import utils
log = logger(__name__)
@click.version_option(nlib.__version__)
@click.group()
def cli():
"""CSV Operations Tool
"""
@cli.command("cvsops")
@click.option('--file', help='Name of csv file')
@click.option('--groupby', help='GroupBy Column Name')
@click.option('--applyname', help='Apply Column Name')
@click.option('--func', help='Appliable Function')
def agg(file,groupby, applyname, func):
"""Operates on a groupby column in a csv file and applies a function
Example Usage:
import sys, os
import signal
from indexer import index_books
from conf import configure
keep_processing = True
os.environ["LOGGING_DEBUG"] = "1"
from sensible.loginit import logger
log = logger(os.path.basename(__file__))
def main(path):
index_path = configure()
ids = [n for n in os.listdir(path) if not n.startswith('.')]
ids.sort()
#for id in ids:
msg = index_books(index_path, ids)
log.info(msg)
#if not keep_processing:
# break
if __name__=='__main__':
if len(sys.argv)>1:
main(sys.argv[1])
else:
print "Usage: %s repo_path " % sys.argv[0]
import os, sys
import pysvn
from library import configure
from sensible.loginit import logger
log = logger(__name__)
class SvnCommand(object):
def __init__(self, path):
self.client = pysvn.Client()
self.path = path
self.revision_update_complete = None
self.client.callback_get_login = self.callback_getLogin
self.client.callback_notify = self.callback_notify
self.client.callback_cancel = self.callback_cancel
def extend_path(self, path):
if path:
self.path = os.path.join(self.path, path)
def get_url(self):
path = self.path
if not os.path.exists(path):
path = os.path.dirname(path)
entries = self.client.info2(path, revision='head', recurse=False)
p, info = entries[0]
return info.URL
"""
Fetch Repositories From Github
"""
import os
from github import Github
import git
from git import GitCommandError
from sensible.loginit import logger
log = logger("devml")
def auth_github(oath_token):
ghub = Github(oath_token)
return ghub
def get_org(oath_token, org):
ghub = auth_github(oath_token)
org = ghub.get_organization(org)
return org
def org_repo_names(oath_token, org):
""""Get Repos for a Github Organization"""
repos = {}
ghub = get_org(oath_token, org)
github_repos = ghub.get_repos()
count = 0
import os
import jsonrpc
import socket
from library import configure
from svnup import update
from invalidate import invalidate
from catalog import index
from sensible.loginit import logger
log = logger(os.path.basename(__file__))
class Dispatcher(object):
def __init__(self, broker_url):
self.broker = jsonrpc.Client(broker_url)
def run(self):
msg = cmd = ''
try:
msg = self.broker.send()
except socket.error, e:
log.error(str(e))
if msg:
log.debug('message:', msg)
cmd, arg = msg.split(':')
if cmd:
cmd_name = 'do_%s' % cmd
m = getattr(self, cmd_name, None)
import boto3
from sensible.loginit import logger
log = logger("My_AWS_setup")
def boto_s3_resource():
"""Create boto3 S3 Resource"""
return boto3.resource("s3")
def download(bucket, key, filename, resource=None):
"""Downloads file from S3"""
if resource is None:
resource = boto_s3_resource()
log_msg = f"Attempting download : {bucket}, {key}, {filename}"
log.info(log_msg)
resource.meta.client.download_file(bucket, key, filename)
return filename