service = get_storage()
req = service.objects().get(
bucket=kwargs['bucket'],
object=filename_remote)
try:
req.execute()
return True
except HttpError as e:
if e.resp.status == 404:
return False
else:
raise
def pull(filename_remote, **kwargs):
service = get_storage()
req = service.objects().get(
object=filename_remote,
bucket=kwargs['bucket'])
resp = req.execute()
print(resp)
files = []
for key in keys:
logging.info("S3 Load: %s", key)
files.append(ReadLineWrapper(key))
return files
BabeBase.addProtocolPushPlugin('gs', push, None, check_exists)
BabeBase.addProtocolPullPlugin('gs', pull)
def build_host(kwargs):
host = kwargs['host']
if 'port' in kwargs:
host = host + ':' + str(kwargs['port'])
if 'user' in kwargs:
host = kwargs['user'] + ':' + kwargs['password'] + '@' + host
return host
def push(filename_topush, filename_remote, **kwargs):
req = urllib2.Request()
req.add_data
host = build_host(kwargs)
f = open(filename_topush, 'rb')
urllib2.urlopen(url='%s://%s/%s' %
(kwargs['protocol'], host, filename_remote),
data=f)
f.close()
def pull(filename_remote, **kwargs):
host = build_host(kwargs)
url = '%s://%s/%s' % (kwargs['protocol'], host, filename_remote)
return urllib2.urlopen(url)
BabeBase.addProtocolPushPlugin('http', push, None)
BabeBase.addProtocolPullPlugin('http', pull)
BabeBase.addProtocolPushPlugin('https', push, None)
BabeBase.addProtocolPullPlugin('https', pull)
def progress_call_back(done, todo):
print "Done %u out of %u (%f%%)" % (done, todo, (done*100)/todo)
def pull(filename_remote, **kwargs):
bucket = get_bucket(kwargs)
cache = BabeBase.get_config("s3", "cache", default=False)
fail_on_empty = kwargs.get("fail_on_empty", True)
if cache:
default_cache_dir = "/tmp/pybabe-s3-cache-%s" % os.getenv('USER')
cache_dir = BabeBase.get_config("s3", "cache_dir", default=default_cache_dir)
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
keys = get_keys(bucket, filename_remote, fail_on_empty=fail_on_empty)
files = []
for key in keys:
logging.info("S3 Load: %s", key)
if cache:
f = os.path.join(cache_dir, os.path.basename(key.name) + "-" + key.etag.replace('"', ''))
if os.path.exists(f):
files.append(open(f, "r"))
else:
key.get_contents_to_filename(f + ".tmp", cb=progress_call_back)
os.rename(f + ".tmp", f)
files.append(open(f, "r"))
else:
files.append(ReadLineWrapper(key))
return files
BabeBase.addProtocolPushPlugin('s3', push, None, check_exists)
BabeBase.addProtocolPullPlugin('s3', pull)
import urllib2
def build_host(kwargs):
host = kwargs['host']
if 'port' in kwargs:
host = host + ':' + str(kwargs['port'])
if 'user' in kwargs:
host = kwargs['user'] + ':' + kwargs['password'] + '@' + host
return host
def push(filename_topush, filename_remote, **kwargs):
req = urllib2.Request()
req.add_data
host = build_host(kwargs)
f = open(filename_topush, 'rb')
urllib2.urlopen(url='%s://%s/%s' % (kwargs['protocol'], host, filename_remote), data=f)
f.close()
def pull(filename_remote, **kwargs):
host = build_host(kwargs)
url = '%s://%s/%s' % (kwargs['protocol'], host, filename_remote)
return urllib2.urlopen(url)
BabeBase.addProtocolPushPlugin('http', push, None)
BabeBase.addProtocolPullPlugin('http', pull)
BabeBase.addProtocolPushPlugin('https', push, None)
BabeBase.addProtocolPullPlugin('https', pull)
def push(filename_topush, filename_remote, **kwargs):
from ftplib import FTP
ftp = FTP()
ftp.connect(kwargs['host'], kwargs.get('port', None))
ftp.login(kwargs.get('user', None), kwargs.get('password', None))
f = open(filename_topush, 'rb')
ftp.storbinary('STOR %s' % filename_remote, f)
f.close()
ftp.quit()
def early_check(**kwargs):
from ftplib import FTP
ftp = FTP()
ftp.connect(kwargs['host'], kwargs.get('port', None))
ftp.login(kwargs.get('user', None), kwargs.get('password', None))
ftp.quit()
def pull(filename_remote, **kwargs):
host = kwargs['host']
if 'port' in kwargs:
host = host + ':' + str(kwargs['port'])
if 'user' in kwargs:
host = kwargs['user'] + ':' + kwargs['password'] + '@' + host
return urllib.urlopen('ftp://%s/%s' % (host, filename_remote))
BabeBase.addProtocolPushPlugin('ftp', push, early_check)
BabeBase.addProtocolPullPlugin('ftp', pull)
