Пример #1
0
"""
S3 methods for PAWS library
"""

import boto3
from sensible.loginit import logger

log = logger("Paws")


def boto_s3_resource():
    """Create boto s3 resource"""
    return boto3.resource("s3")


def download(bucket, key, filename, resource=None):
    """Downloads file from S3"""
    if resource is None:
        resource = boto_s3_resource()

    log_msg = "Attempting to download {}, {}, {}".format(bucket, key, filename)

    log.info(log_msg)

    resource.meta.client.download_file(bucket, key, filename)

    return filename
Пример #2
0
"""
Commandline Tool For Doing CSV operations:
    * Aggregation
    * TBD
"""

import sys

import click
from sensible.loginit import logger

import nlib
from nlib import csvops
from nlib import utils

log = logger(__name__)


@click.version_option(nlib.__version__)
@click.group()
def cli():
    """CSV Operations Tool
    """
@cli.command("cvsops")
@click.option('--file', help='Name of csv file')
@click.option('--groupby', help='GroupBy Column Name')
@click.option('--applyname', help='Apply Column Name')
@click.option('--func', help='Appliable Function')
def agg(file,groupby, applyname, func):
    """Operates on a groupby column in a csv file and applies a function
    Example Usage:
Пример #3
0
import sys, os
import signal

from indexer import index_books
from conf import configure

keep_processing = True

os.environ["LOGGING_DEBUG"] =  "1"
from sensible.loginit import logger
log = logger(os.path.basename(__file__))


def main(path):
    index_path = configure()
    ids = [n for n in os.listdir(path) if not n.startswith('.')]
    ids.sort()
    
    #for id in ids:
    msg = index_books(index_path, ids)
    log.info(msg)
    #if not keep_processing:
    #    break
    
if __name__=='__main__':
    if len(sys.argv)>1:
        main(sys.argv[1])
    else:
        print "Usage: %s repo_path " % sys.argv[0]
Пример #4
0
import os, sys
import pysvn
from library import configure

from sensible.loginit import logger

log = logger(__name__)

class SvnCommand(object):
    
    def __init__(self, path):
        self.client = pysvn.Client()
        self.path = path
        self.revision_update_complete = None
        self.client.callback_get_login = self.callback_getLogin
        self.client.callback_notify = self.callback_notify
        self.client.callback_cancel = self.callback_cancel
    
    def extend_path(self, path):
        if path:
            self.path = os.path.join(self.path, path)
            
    def get_url(self):
        path = self.path 
        if not os.path.exists(path):
            path = os.path.dirname(path)
            
        entries = self.client.info2(path, revision='head', recurse=False)
        p, info = entries[0]
        return info.URL
            
Пример #5
0
"""
Fetch Repositories From Github
"""

import os

from github import Github
import git
from git import GitCommandError

from sensible.loginit import logger
log = logger("devml")

def auth_github(oath_token):
    
    ghub = Github(oath_token)
    return ghub

def get_org(oath_token, org):

    ghub = auth_github(oath_token)
    org = ghub.get_organization(org)
    return org

def org_repo_names(oath_token, org):
    """"Get Repos for a Github Organization"""
    
    repos = {}
    ghub = get_org(oath_token, org)
    github_repos = ghub.get_repos()
    count = 0
Пример #6
0
import os
import jsonrpc
import socket
from library import configure

from svnup import update
from invalidate import invalidate
from catalog import index

from sensible.loginit import logger

log = logger(os.path.basename(__file__))


class Dispatcher(object):
    def __init__(self, broker_url):
        self.broker = jsonrpc.Client(broker_url)

    def run(self):
        msg = cmd = ''
        try:
            msg = self.broker.send()
        except socket.error, e:
            log.error(str(e))

        if msg:
            log.debug('message:', msg)
            cmd, arg = msg.split(':')
            if cmd:
                cmd_name = 'do_%s' % cmd
                m = getattr(self, cmd_name, None)
Пример #7
0
import boto3
from sensible.loginit import logger

log = logger("My_AWS_setup")


def boto_s3_resource():
    """Create boto3 S3 Resource"""
    return boto3.resource("s3")


def download(bucket, key, filename, resource=None):
    """Downloads file from S3"""
    if resource is None:
        resource = boto_s3_resource()
    log_msg = f"Attempting download : {bucket}, {key}, {filename}"
    log.info(log_msg)
    resource.meta.client.download_file(bucket, key, filename)
    return filename