def download_thread():
try:
cherrypy.log("Thread Start User: {}".format(user))
try:
session = newman_email.login(user, passwd, logfile)
fldr = "{}/emails/{}".format(webroot, user)
cherrypy.log("Login User: {}".format(user))
if os.path.exists(fldr):
rmrf(fldr)
mkdir(fldr)
spit("{}/output.csv".format(fldr),
newman_email.headerrow() + "\n")
mkdir(fldr + "/emails")
newman_email.download(session, user, fldr, int(limit), logfile)
spit(logfile, "[Completed Download] {}\n".format(user))
except Exception as ex:
spit(logfile, "[Error] {}\n".format(ex))
cherrypy.log("[Error] {}\n".format(ex))
except:
spit(logfile, "[Error]")
cherrypy.log("[Error]")
error_info = sys.exc_info()[0]
cherrypy.log(error_info)
spit(logfile,
"[Error] {}\n".format(error_info.replace('\n', ' ')))
finally:
newman_email.close_session(session)
except:
error_info = sys.exc_info()[0]
cherrypy.log(error_info)
spit(logfile, "[Error] {}\n".format(error_info.replace('\n', ' ')))
def download_thread():
try:
cherrypy.log("Thread Start User: {}".format(user))
try:
session = newman_email.login(user, passwd, logfile)
fldr = "{}/emails/{}".format(webroot, user)
cherrypy.log("Login User: {}".format(user))
if os.path.exists(fldr):
rmrf(fldr)
mkdir(fldr)
spit("{}/output.csv".format(fldr), newman_email.headerrow() + "\n")
mkdir(fldr + "/emails")
newman_email.download(session, user, fldr, int(limit), logfile)
spit(logfile, "[Completed Download] {}\n".format(user))
except Exception as ex:
spit(logfile, "[Error] {}\n".format(ex))
cherrypy.log("[Error] {}\n".format(ex))
except:
spit(logfile, "[Error]")
cherrypy.log("[Error]")
error_info = sys.exc_info()[0]
cherrypy.log(error_info)
spit(logfile, "[Error] {}\n".format(error_info.replace('\n', ' ')))
finally:
newman_email.close_session(session)
except:
error_info = sys.exc_info()[0]
cherrypy.log(error_info)
spit(logfile, "[Error] {}\n".format(error_info.replace('\n', ' ')))
def buildExportable(*args):
webroot = cherrypy.config.get("webroot")
target = getOpt('target')
base_src = "{}/emails/{}".format(webroot, target)
tmp_dir = os.path.abspath("{}/../tmp/".format(webroot))
download_dir = "{}/downloads/".format(webroot)
tar_gz = "export_{}".format(fmtNow())
base_dest = os.path.abspath("{}/../tmp/newman_dl".format(webroot))
if os.path.exists(base_dest):
rmrf(base_dest)
if not os.path.exists(download_dir):
mkdir(download_dir)
mkdir(base_dest)
# Get list of paths...
stmt = (" SELECT id, dir FROM email WHERE exportable='true' ")
msg = ''
paths_to_copy = []
tangelo.content_type("application/json")
with newman_connector() as read_cnx:
with execute_query(read_cnx.conn(), stmt) as qry:
for email_id, val in qry.cursor():
src = "{}/{}/".format(base_src, val)
dest = "{}/{}/".format(base_dest, val)
shutil.copytree(src, dest)
# compress dir
shutil.make_archive("{}/{}".format(tmp_dir, tar_gz),
"gztar",
root_dir=base_dest)
# move to web downloads
mv("{}/{}.tar.gz".format(tmp_dir, tar_gz),
"{}/{}.tar.gz".format(download_dir, tar_gz))
return {"file": "downloads/{}.tar.gz".format(tar_gz)}
def buildExportable(*args):
webroot = cherrypy.config.get("webroot")
target = getOpt('target')
base_src = "{}/emails/{}".format(webroot,target)
tmp_dir = os.path.abspath("{}/../tmp/".format(webroot))
download_dir = "{}/downloads/".format(webroot)
tar_gz = "export_{}".format(fmtNow())
base_dest = os.path.abspath("{}/../tmp/newman_dl".format(webroot))
if os.path.exists(base_dest):
rmrf(base_dest)
if not os.path.exists(download_dir):
mkdir(download_dir)
mkdir(base_dest)
# Get list of paths...
stmt = (
" SELECT id, dir FROM email WHERE exportable='true' "
)
msg = ''
paths_to_copy = []
tangelo.content_type("application/json")
with newman_connector() as read_cnx:
with execute_query(read_cnx.conn(), stmt) as qry:
for email_id, val in qry.cursor():
src = "{}/{}/".format(base_src,val)
dest = "{}/{}/".format(base_dest, val)
shutil.copytree(src, dest)
# compress dir
shutil.make_archive("{}/{}".format(tmp_dir, tar_gz), "gztar", root_dir=base_dest)
# move to web downloads
mv("{}/{}.tar.gz".format(tmp_dir, tar_gz), "{}/{}.tar.gz".format(download_dir, tar_gz))
return { "file" : "downloads/{}.tar.gz".format(tar_gz) }
def download(srv, target_email, outdir, limit, logfile):
srv.select("[Gmail]/All Mail", True)
#resp, data = srv.uid('SEARCH', None, 'ALL')
resp, data = srv.search(None, 'ALL')
if resp != 'OK':
err_msg = "Error searching: %s %s" % (resp, data)
spit(logfile, "[Error] {}\n".format(err_msg))
raise Exception(err_msg)
msgids = data[0].split()
if limit > 0:
msgids = msgids[-limit:]
attach_count = counter()
c = counter()
l = len(msgids)
for msgid in msgids:
try:
uid = getUIDForMessage(srv, msgid)
fldr ="emails/{}".format(uid)
mkdir("{}/{}".format(outdir, fldr))
i = c.next()
if i % 200 == 0:
spit(logfile, "[Downloading] Downloaded: {}/{}\n".format(i,l))
resp, msgParts = srv.fetch(msgid, '(RFC822)')
if resp != 'OK':
err_msg = "Bad response: %s %s" % (resp, msgParts)
spit(logfile, "[Error] {}\n".format(err_msg))
raise Exception(err_msg)
emailBody = msgParts[0][1]
spit("{}/{}/{}.eml".format(outdir,fldr, uid), emailBody)
mail = email.message_from_string(emailBody)
attach = []
msg=""
for part in mail.walk():
if part.get_content_type() == 'text/plain':
msg = msg + "\n" + part.get_payload()
if part.get_content_maintype() == 'multipart':
continue
if part.get('Content-Disposition') is None:
continue
fileName = part.get_filename()
#escape file name
fileName = fileName if fileName else "Attach_{}".format(attach_count.next())
fileName = fileName.replace('/','_')
attach.append(fileName)
filePath = "{}/{}/{}".format(outdir, fldr, fileName)
fp = open(filePath, 'wb')
fp.write(part.get_payload(decode=True))
fp.close()
msg = re.sub(r'[^\x00-\x7F]',' ', msg)
spit("{}/{}/{}.txt".format(outdir,fldr, uid), msg)
row = createRow(uid, fldr, target_email, mail, attach, msg)
spit("{}/output.csv".format(outdir), row + "\n")
except Exception, e:
spit(logfile, "[Downloading] [Exception]: line {}, msgid {}, except {}\n".format(i,msgid, str(e)))
continue
def download(srv, target_email, outdir, limit, logfile):
srv.select("[Gmail]/All Mail", True)
#resp, data = srv.uid('SEARCH', None, 'ALL')
resp, data = srv.search(None, 'ALL')
if resp != 'OK':
err_msg = "Error searching: %s %s" % (resp, data)
spit(logfile, "[Error] {}\n".format(err_msg))
raise Exception(err_msg)
msgids = data[0].split()
if limit > 0:
msgids = msgids[-limit:]
attach_count = counter()
c = counter()
l = len(msgids)
for msgid in msgids:
try:
uid = getUIDForMessage(srv, msgid)
fldr = "emails/{}".format(uid)
mkdir("{}/{}".format(outdir, fldr))
i = c.next()
if i % 200 == 0:
spit(logfile, "[Downloading] Downloaded: {}/{}\n".format(i, l))
resp, msgParts = srv.fetch(msgid, '(RFC822)')
if resp != 'OK':
err_msg = "Bad response: %s %s" % (resp, msgParts)
spit(logfile, "[Error] {}\n".format(err_msg))
raise Exception(err_msg)
emailBody = msgParts[0][1]
spit("{}/{}/{}.eml".format(outdir, fldr, uid), emailBody)
mail = email.message_from_string(emailBody)
attach = []
msg = ""
for part in mail.walk():
if part.get_content_type() == 'text/plain':
msg = msg + "\n" + part.get_payload()
if part.get_content_maintype() == 'multipart':
continue
if part.get('Content-Disposition') is None:
continue
fileName = part.get_filename()
#escape file name
fileName = fileName if fileName else "Attach_{}".format(
attach_count.next())
fileName = fileName.replace('/', '_')
attach.append(fileName)
filePath = "{}/{}/{}".format(outdir, fldr, fileName)
fp = open(filePath, 'wb')
fp.write(part.get_payload(decode=True))
fp.close()
msg = re.sub(r'[^\x00-\x7F]', ' ', msg)
spit("{}/{}/{}.txt".format(outdir, fldr, uid), msg)
row = createRow(uid, fldr, target_email, mail, attach, msg)
spit("{}/output.csv".format(outdir), row + "\n")
except Exception, e:
spit(
logfile,
"[Downloading] [Exception]: line {}, msgid {}, except {}\n".
format(i, msgid, str(e)))
continue
def setup(_dir):
if os.path.exists(_dir):
rmrf(_dir)
mkdir(_dir)
def setup(_dir):
if os.path.exists(_dir):
rmrf(_dir)
mkdir(_dir)
