from base import BabeBase
import urllib2
import urllib
import json
def pull_buzz(stream, username, dataroom, uuid, **kwargs):
url = 'https://buzzdata.com/api/%s/%s/%s/download_request' % (username, dataroom, uuid)
if 'api_key' in kwargs:
api_key = kwargs['api_key']
elif BabeBase.get_config('buzzdata', 'api_key'):
api_key = BabeBase.get_config('buzzdata', 'api_key')
else:
raise Exception('Missing api_key')
data = urllib.urlencode([('api_key', api_key)])
drequest = urllib2.urlopen(url, data).read()
obj = json.loads(drequest)
download_url = obj['download_request']['url']
return urllib2.urlopen(download_url)
BabeBase.addProtocolPullPlugin('buzzdata', pull_buzz)
service = get_storage()
req = service.objects().get(
bucket=kwargs['bucket'],
object=filename_remote)
try:
req.execute()
return True
except HttpError as e:
if e.resp.status == 404:
return False
else:
raise
def pull(filename_remote, **kwargs):
service = get_storage()
req = service.objects().get(
object=filename_remote,
bucket=kwargs['bucket'])
resp = req.execute()
print(resp)
files = []
for key in keys:
logging.info("S3 Load: %s", key)
files.append(ReadLineWrapper(key))
return files
BabeBase.addProtocolPushPlugin('gs', push, None, check_exists)
BabeBase.addProtocolPullPlugin('gs', pull)
def build_host(kwargs):
host = kwargs['host']
if 'port' in kwargs:
host = host + ':' + str(kwargs['port'])
if 'user' in kwargs:
host = kwargs['user'] + ':' + kwargs['password'] + '@' + host
return host
def push(filename_topush, filename_remote, **kwargs):
req = urllib2.Request()
req.add_data
host = build_host(kwargs)
f = open(filename_topush, 'rb')
urllib2.urlopen(url='%s://%s/%s' %
(kwargs['protocol'], host, filename_remote),
data=f)
f.close()
def pull(filename_remote, **kwargs):
host = build_host(kwargs)
url = '%s://%s/%s' % (kwargs['protocol'], host, filename_remote)
return urllib2.urlopen(url)
BabeBase.addProtocolPushPlugin('http', push, None)
BabeBase.addProtocolPullPlugin('http', pull)
BabeBase.addProtocolPushPlugin('https', push, None)
BabeBase.addProtocolPullPlugin('https', pull)
from base import BabeBase
import urllib2
import urllib
import json
def pull_buzz(stream, username, dataroom, uuid, **kwargs):
url = 'https://buzzdata.com/api/%s/%s/%s/download_request' % (
username, dataroom, uuid)
if 'api_key' in kwargs:
api_key = kwargs['api_key']
elif BabeBase.get_config('buzzdata', 'api_key'):
api_key = BabeBase.get_config('buzzdata', 'api_key')
else:
raise Exception('Missing api_key')
data = urllib.urlencode([('api_key', api_key)])
drequest = urllib2.urlopen(url, data).read()
obj = json.loads(drequest)
download_url = obj['download_request']['url']
return urllib2.urlopen(download_url)
BabeBase.addProtocolPullPlugin('buzzdata', pull_buzz)
import urllib2
def build_host(kwargs):
host = kwargs['host']
if 'port' in kwargs:
host = host + ':' + str(kwargs['port'])
if 'user' in kwargs:
host = kwargs['user'] + ':' + kwargs['password'] + '@' + host
return host
def push(filename_topush, filename_remote, **kwargs):
req = urllib2.Request()
req.add_data
host = build_host(kwargs)
f = open(filename_topush, 'rb')
urllib2.urlopen(url='%s://%s/%s' % (kwargs['protocol'], host, filename_remote), data=f)
f.close()
def pull(filename_remote, **kwargs):
host = build_host(kwargs)
url = '%s://%s/%s' % (kwargs['protocol'], host, filename_remote)
return urllib2.urlopen(url)
BabeBase.addProtocolPushPlugin('http', push, None)
BabeBase.addProtocolPullPlugin('http', pull)
BabeBase.addProtocolPushPlugin('https', push, None)
BabeBase.addProtocolPullPlugin('https', pull)
from base import BabeBase
import urllib2, urllib, json
def pull_buzz(stream, username, dataroom, uuid, **kwargs):
url = "https://buzzdata.com/api/%s/%s/%s/download_request" % (username, dataroom, uuid)
if "api_key" in kwargs:
api_key = kwargs["api_key"]
elif BabeBase.get_config("buzzdata", "api_key"):
api_key = BabeBase.get_config("buzzdata", "api_key")
else:
raise Exception("Missing api_key")
data = urllib.urlencode([("api_key", api_key)])
drequest = urllib2.urlopen(url, data).read()
obj = json.loads(drequest)
download_url = obj["download_request"]["url"]
return urllib2.urlopen(download_url)
BabeBase.addProtocolPullPlugin("buzzdata", pull_buzz)
def progress_call_back(done, todo):
print "Done %u out of %u (%f%%)" % (done, todo, (done*100)/todo)
def pull(filename_remote, **kwargs):
bucket = get_bucket(kwargs)
cache = BabeBase.get_config("s3", "cache", default=False)
fail_on_empty = kwargs.get("fail_on_empty", True)
if cache:
default_cache_dir = "/tmp/pybabe-s3-cache-%s" % os.getenv('USER')
cache_dir = BabeBase.get_config("s3", "cache_dir", default=default_cache_dir)
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
keys = get_keys(bucket, filename_remote, fail_on_empty=fail_on_empty)
files = []
for key in keys:
logging.info("S3 Load: %s", key)
if cache:
f = os.path.join(cache_dir, os.path.basename(key.name) + "-" + key.etag.replace('"', ''))
if os.path.exists(f):
files.append(open(f, "r"))
else:
key.get_contents_to_filename(f + ".tmp", cb=progress_call_back)
os.rename(f + ".tmp", f)
files.append(open(f, "r"))
else:
files.append(ReadLineWrapper(key))
return files
BabeBase.addProtocolPushPlugin('s3', push, None, check_exists)
BabeBase.addProtocolPullPlugin('s3', pull)
