from base import BabeBase import urllib2 import urllib import json def pull_buzz(stream, username, dataroom, uuid, **kwargs): url = 'https://buzzdata.com/api/%s/%s/%s/download_request' % (username, dataroom, uuid) if 'api_key' in kwargs: api_key = kwargs['api_key'] elif BabeBase.get_config('buzzdata', 'api_key'): api_key = BabeBase.get_config('buzzdata', 'api_key') else: raise Exception('Missing api_key') data = urllib.urlencode([('api_key', api_key)]) drequest = urllib2.urlopen(url, data).read() obj = json.loads(drequest) download_url = obj['download_request']['url'] return urllib2.urlopen(download_url) BabeBase.addProtocolPullPlugin('buzzdata', pull_buzz)
service = get_storage() req = service.objects().get( bucket=kwargs['bucket'], object=filename_remote) try: req.execute() return True except HttpError as e: if e.resp.status == 404: return False else: raise def pull(filename_remote, **kwargs): service = get_storage() req = service.objects().get( object=filename_remote, bucket=kwargs['bucket']) resp = req.execute() print(resp) files = [] for key in keys: logging.info("S3 Load: %s", key) files.append(ReadLineWrapper(key)) return files BabeBase.addProtocolPushPlugin('gs', push, None, check_exists) BabeBase.addProtocolPullPlugin('gs', pull)
def build_host(kwargs): host = kwargs['host'] if 'port' in kwargs: host = host + ':' + str(kwargs['port']) if 'user' in kwargs: host = kwargs['user'] + ':' + kwargs['password'] + '@' + host return host def push(filename_topush, filename_remote, **kwargs): req = urllib2.Request() req.add_data host = build_host(kwargs) f = open(filename_topush, 'rb') urllib2.urlopen(url='%s://%s/%s' % (kwargs['protocol'], host, filename_remote), data=f) f.close() def pull(filename_remote, **kwargs): host = build_host(kwargs) url = '%s://%s/%s' % (kwargs['protocol'], host, filename_remote) return urllib2.urlopen(url) BabeBase.addProtocolPushPlugin('http', push, None) BabeBase.addProtocolPullPlugin('http', pull) BabeBase.addProtocolPushPlugin('https', push, None) BabeBase.addProtocolPullPlugin('https', pull)
from base import BabeBase import urllib2 import urllib import json def pull_buzz(stream, username, dataroom, uuid, **kwargs): url = 'https://buzzdata.com/api/%s/%s/%s/download_request' % ( username, dataroom, uuid) if 'api_key' in kwargs: api_key = kwargs['api_key'] elif BabeBase.get_config('buzzdata', 'api_key'): api_key = BabeBase.get_config('buzzdata', 'api_key') else: raise Exception('Missing api_key') data = urllib.urlencode([('api_key', api_key)]) drequest = urllib2.urlopen(url, data).read() obj = json.loads(drequest) download_url = obj['download_request']['url'] return urllib2.urlopen(download_url) BabeBase.addProtocolPullPlugin('buzzdata', pull_buzz)
import urllib2 def build_host(kwargs): host = kwargs['host'] if 'port' in kwargs: host = host + ':' + str(kwargs['port']) if 'user' in kwargs: host = kwargs['user'] + ':' + kwargs['password'] + '@' + host return host def push(filename_topush, filename_remote, **kwargs): req = urllib2.Request() req.add_data host = build_host(kwargs) f = open(filename_topush, 'rb') urllib2.urlopen(url='%s://%s/%s' % (kwargs['protocol'], host, filename_remote), data=f) f.close() def pull(filename_remote, **kwargs): host = build_host(kwargs) url = '%s://%s/%s' % (kwargs['protocol'], host, filename_remote) return urllib2.urlopen(url) BabeBase.addProtocolPushPlugin('http', push, None) BabeBase.addProtocolPullPlugin('http', pull) BabeBase.addProtocolPushPlugin('https', push, None) BabeBase.addProtocolPullPlugin('https', pull)
from base import BabeBase import urllib2, urllib, json def pull_buzz(stream, username, dataroom, uuid, **kwargs): url = "https://buzzdata.com/api/%s/%s/%s/download_request" % (username, dataroom, uuid) if "api_key" in kwargs: api_key = kwargs["api_key"] elif BabeBase.get_config("buzzdata", "api_key"): api_key = BabeBase.get_config("buzzdata", "api_key") else: raise Exception("Missing api_key") data = urllib.urlencode([("api_key", api_key)]) drequest = urllib2.urlopen(url, data).read() obj = json.loads(drequest) download_url = obj["download_request"]["url"] return urllib2.urlopen(download_url) BabeBase.addProtocolPullPlugin("buzzdata", pull_buzz)
def progress_call_back(done, todo): print "Done %u out of %u (%f%%)" % (done, todo, (done*100)/todo) def pull(filename_remote, **kwargs): bucket = get_bucket(kwargs) cache = BabeBase.get_config("s3", "cache", default=False) fail_on_empty = kwargs.get("fail_on_empty", True) if cache: default_cache_dir = "/tmp/pybabe-s3-cache-%s" % os.getenv('USER') cache_dir = BabeBase.get_config("s3", "cache_dir", default=default_cache_dir) if not os.path.exists(cache_dir): os.makedirs(cache_dir) keys = get_keys(bucket, filename_remote, fail_on_empty=fail_on_empty) files = [] for key in keys: logging.info("S3 Load: %s", key) if cache: f = os.path.join(cache_dir, os.path.basename(key.name) + "-" + key.etag.replace('"', '')) if os.path.exists(f): files.append(open(f, "r")) else: key.get_contents_to_filename(f + ".tmp", cb=progress_call_back) os.rename(f + ".tmp", f) files.append(open(f, "r")) else: files.append(ReadLineWrapper(key)) return files BabeBase.addProtocolPushPlugin('s3', push, None, check_exists) BabeBase.addProtocolPullPlugin('s3', pull)