def pull(filename_remote, **kwargs):
bucket = get_bucket(kwargs)
cache = BabeBase.get_config("s3", "cache", default=False)
fail_on_empty = kwargs.get("fail_on_empty", True)
if cache:
default_cache_dir = "/tmp/pybabe-s3-cache-%s" % os.getenv('USER')
cache_dir = BabeBase.get_config("s3",
"cache_dir",
default=default_cache_dir)
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
keys = get_keys(bucket, filename_remote, fail_on_empty=fail_on_empty)
files = []
for key in keys:
logging.info("S3 Load: %s", key)
if cache:
f = os.path.join(
cache_dir,
os.path.basename(key.name) + "-" + key.etag.replace('"', ''))
if os.path.exists(f):
files.append(open(f, "r"))
else:
key.get_contents_to_filename(f + ".tmp")
os.rename(f + ".tmp", f)
files.append(open(f, "r"))
else:
files.append(ReadLineWrapper(key))
return files
def pull(filename_remote, **kwargs):
bucket = get_bucket(kwargs)
cache = BabeBase.get_config("s3", "cache", default=False)
fail_on_empty = kwargs.get("fail_on_empty", True)
if cache:
default_cache_dir = "/tmp/pybabe-s3-cache-%s" % os.getenv('USER')
cache_dir = BabeBase.get_config("s3", "cache_dir", default=default_cache_dir)
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
keys = get_keys(bucket, filename_remote, fail_on_empty=fail_on_empty)
files = []
for key in keys:
logging.info("S3 Load: %s", key)
if cache:
f = os.path.join(cache_dir,
os.path.basename(key.name) + "-" + key.etag.replace('"', ''))
if os.path.exists(f):
files.append(open(f, "r"))
else:
key.get_contents_to_filename(f + ".tmp", cb=progress_call_back)
os.rename(f + ".tmp", f)
files.append(open(f, "r"))
else:
files.append(ReadLineWrapper(key))
return files
def __init__(self, **kwargs):
self.size_limit = kwargs.get('size_limit',5<<30) #1<<20 (1MB), 1<<30 (1GB)
self.cache_directories = []
cache = BabeBase.get_config("s3", "cache", default=False)
if cache:
default_cache_dir = "/tmp/pybabe-s3-cache-%s" % os.getenv('USER')
cache_dir = BabeBase.get_config("s3", "cache_dir", default=default_cache_dir)
self.cache_directories.append(cache_dir)
self.cache_directories.append(BabeBase.get_config_with_env(section='kontagent', key='KT_FILECACHE', default='/tmp/kontagent-cache'))
def pull_buzz(stream, username, dataroom, uuid, **kwargs):
url = 'https://buzzdata.com/api/%s/%s/%s/download_request' % (username, dataroom, uuid)
if 'api_key' in kwargs:
api_key = kwargs['api_key']
elif BabeBase.get_config('buzzdata', 'api_key'):
api_key = BabeBase.get_config('buzzdata', 'api_key')
else:
raise Exception('Missing api_key')
data = urllib.urlencode([('api_key', api_key)])
drequest = urllib2.urlopen(url, data).read()
obj = json.loads(drequest)
download_url = obj['download_request']['url']
return urllib2.urlopen(download_url)
def pull_buzz(stream, username, dataroom, uuid, **kwargs):
url = "https://buzzdata.com/api/%s/%s/%s/download_request" % (username, dataroom, uuid)
if "api_key" in kwargs:
api_key = kwargs["api_key"]
elif BabeBase.get_config("buzzdata", "api_key"):
api_key = BabeBase.get_config("buzzdata", "api_key")
else:
raise Exception("Missing api_key")
data = urllib.urlencode([("api_key", api_key)])
drequest = urllib2.urlopen(url, data).read()
obj = json.loads(drequest)
download_url = obj["download_request"]["url"]
return urllib2.urlopen(download_url)
def pull_buzz(stream, username, dataroom, uuid, **kwargs):
url = 'https://buzzdata.com/api/%s/%s/%s/download_request' % (
username, dataroom, uuid)
if 'api_key' in kwargs:
api_key = kwargs['api_key']
elif BabeBase.get_config('buzzdata', 'api_key'):
api_key = BabeBase.get_config('buzzdata', 'api_key')
else:
raise Exception('Missing api_key')
data = urllib.urlencode([('api_key', api_key)])
drequest = urllib2.urlopen(url, data).read()
obj = json.loads(drequest)
download_url = obj['download_request']['url']
return urllib2.urlopen(download_url)
def mail(stream,
subject,
recipients,
in_body=False,
in_body_row_limit=None,
attach_formats="csv",
**kwargs):
"""Format a stream in a mail and send it.
Recipients: list of recipients mail addresses
in_body: format (in HTML & text) the content
in_body_row_limit : maximum number of line in body
attach_format : file format to use for attachment
"""
smtp_server = BabeBase.get_config('smtp', 'server', kwargs)
smtp_port = BabeBase.get_config('smtp', 'port', kwargs)
smtp_tls = BabeBase.get_config('smtp', 'tls', kwargs, False)
smtp_login = BabeBase.get_config('smtp', 'login', kwargs)
smtp_password = BabeBase.get_config('smtp', 'password', kwargs)
author = BabeBase.get_config('smtp', 'author', kwargs)
formats = []
if in_body:
formats.append("html")
if attach_formats:
if isinstance(attach_formats, basestring):
formats.append(attach_formats)
else:
formats.extend(attach_formats)
if isinstance(recipients, basestring):
recipients = [recipients]
babes = stream.tee(len(formats))
if in_body and in_body_row_limit:
babes[0] = babes[0].head(in_body_row_limit, all_streams=True)
buffer_dicts = []
for format, babe in izip(formats, babes):
d = ordered_dict()
babe.push(stream_dict=d, format=format)
buffer_dicts.append((format, d))
msg = MIMEMultipart()
msg['Subject'] = subject
msg['From'] = author
msg['To'] = ', '.join(recipients)
for format, d in buffer_dicts:
if format == "html":
buf = StringIO()
buf.write('<html><body>\n')
for filename in d:
buf.write(d[filename].getvalue())
buf.write('\n')
buf.write('\n</body></html>')
att = MIMEText(buf.getvalue(), "html")
msg.attach(att)
else:
for filename in d:
c = d[filename].getvalue()
(maintype, subtype) = BabeBase.getMimeType(format)
att = MIMEBase(maintype, subtype)
att.set_payload(c)
encoders.encode_base64(att)
att.add_header('Content-Disposition',
'attachment',
filename=filename + "." + format)
msg.attach(att)
s = smtplib.SMTP(smtp_server, smtp_port)
s.ehlo()
if smtp_tls:
s.starttls()
s.ehlo()
s.login(smtp_login, smtp_password)
s.sendmail(author, recipients, msg.as_string())
s.quit()
def mail(stream, subject, recipients, in_body=False, in_body_row_limit=None, attach_formats = "csv", **kwargs):
"""Format a stream in a mail and send it.
Recipients: list of recipients mail addresses
in_body: format (in HTML & text) the content
in_body_row_limit : maximum number of line in body
attach_format : file format to use for attachment
"""
smtp_server = BabeBase.get_config('smtp', 'server', kwargs)
smtp_port = BabeBase.get_config('smtp', 'port', kwargs)
smtp_tls = BabeBase.get_config('smtp', 'tls', kwargs, False)
smtp_login = BabeBase.get_config('smtp', 'login', kwargs)
smtp_password = BabeBase.get_config('smtp', 'password', kwargs)
author = BabeBase.get_config('smtp', 'author', kwargs)
formats = []
if in_body:
formats.append("html")
if attach_formats:
if isinstance(attach_formats, basestring):
formats.append(attach_formats)
else:
formats.extend(attach_formats)
if isinstance(recipients, basestring):
recipients = [recipients]
babes = stream.tee(len(formats))
if in_body and in_body_row_limit:
babes[0] = babes[0].head(in_body_row_limit, all_streams=True)
buffer_dicts = []
for format, babe in izip(formats, babes):
d = ordered_dict()
babe.push(stream_dict=d, format=format)
buffer_dicts.append((format, d))
msg = MIMEMultipart()
msg['Subject'] = subject
msg['From'] = author
msg['To'] = ', '.join(recipients)
for format, d in buffer_dicts:
if format == "html":
buf = StringIO()
buf.write('<html><body>\n')
for filename in d:
buf.write(d[filename].getvalue())
buf.write('\n')
buf.write('\n</body></html>')
att = MIMEText(buf.getvalue(),"html")
msg.attach(att)
else:
for filename in d:
c = d[filename].getvalue()
(maintype, subtype) = BabeBase.getMimeType(format)
att = MIMEBase(maintype, subtype)
att.set_payload(c)
encoders.encode_base64(att)
att.add_header('Content-Disposition', 'attachment', filename=filename + "." + format)
msg.attach(att)
s = smtplib.SMTP(smtp_server, smtp_port)
s.ehlo()
if smtp_tls:
s.starttls()
s.ehlo()
s.login(smtp_login, smtp_password)
s.sendmail(author, recipients, msg.as_string())
s.quit()
