def ldap_connect():
'''a helper method for connecting to ldap'''
try:
ldap_obj = psuldap.psuldap()
ldap_obj.connect(ldapurl='ldap://ldap.oit.pdx.edu/')
ldap_connected = True
return ldap_obj, ldap_connected
except psuldap.ldap.LDAPError, err:
print '\nerror connecting to ldap: {0}\n'.format(err)
return None, False
def ldap_lookup( website_no_http ):
'''helper method for handling the ldap lookup'''
ldap_connected = False
ldap_searched = False
search_results = []
ldap_results = []
try:
ldap_obj = psuldap.psuldap()
ldap_obj.connect(ldapurl='ldap://ldap.oit.pdx.edu/')
ldap_connected = True
except:
print '\nerror connecting to ldap\n'
if ldap_connected:
this_search_filter = '(&(eduPersonAffiliation=WEB)(labeledUri=*' + str( website_no_http ) + '*))'
try:
search_results = ldap_obj.search( searchfilter=this_search_filter )
ldap_searched = True
if len( search_results ) > 0:
ldap_results.append( search_results )
except:
print '\nerror searching ldap\n' + str( exc_info()[1] ) + '\n'
if len( search_results ) is 0:
this_search_filter = '(&(eduPersonAffiliation=WEB)(cn=*' + str( website_no_http ) + '*))'
try:
search_results = ldap_obj.search( searchfilter=this_search_filter )
ldap_searched = True
if len( search_results ) > 0:
ldap_results.append( search_results )
except:
print '\nerror searching ldap\n' + str( exc_info()[1] ) + '\n'
if len( search_results ) is 0:
this_search_filter = '(&(eduPersonAffiliation=WEB)(gecos=*' + str( website_no_http ) + '*))'
try:
search_results = ldap_obj.search( searchfilter=this_search_filter )
ldap_searched = True
if len( search_results ) > 0:
ldap_results.append( search_results )
except:
print '\nerror searching ldap\n' + str( exc_info()[1] ) + '\n'
#ldap searched will always be true provided there was no error
if ldap_searched:
if len( search_results ) is 0:
uid = ''
else:
uid = str( search_results[0][1]['uid'])
else:
uid = ''
return uid, search_results
def route_to_psu_future(self, login):
ldap = psuldap('/vol/certs')
# ldapsearch -x -h ldap.oit.pdx.edu -b 'dc=pdx, dc=edu' uid=dennis mailhost
ldap_host = self.prop.getProperty('ldap.host')
ldap_login = self.prop.getProperty('ldap.login')
ldap_password = self.prop.getProperty('ldap.password')
#self.log.info('opt_in_alread(): connecting to LDAP: ' + ldap_host)
ldap.connect( ldap_host, ldap_login, ldap_password)
dn = 'uid=' + login + ',ou=people,dc=pdx,dc=edu'
ldap.mod_attribute(dn, 'mailHost', 'cyrus.psumail.pdx.edu')
def get_ldap_attr(self, login, attr):
ldap = psuldap('/vol/certs')
ldap_host = self.prop.getProperty('ldap.read.host')
ldap_login = self.prop.getProperty('ldap.login')
ldap_password = self.prop.getProperty('ldap.password')
self.log.info('opt_in_alread(): connecting to LDAP: ' + ldap_host)
ldap.connect( ldap_host, ldap_login, ldap_password)
res = ldap.search( searchfilter = 'uid=' + login, attrlist = ['mailHost'])
for (dn, result) in res:
if result.has_key(attr):
return str(result[attr])
return None
def opt_in_already(self, login):
ldap = psuldap('/vol/certs')
ldap_host = self.prop.getProperty('ldap.read.host')
ldap_login = self.prop.getProperty('ldap.login')
ldap_password = self.prop.getProperty('ldap.password')
self.log.info('opt_in_alread(): connecting to LDAP: ' + ldap_host)
ldap.connect( ldap_host, ldap_login, ldap_password)
res = ldap.search( searchfilter = 'uid=' + login, attrlist = ['mailHost'])
for (dn, result) in res:
if result.has_key("mailHost"):
self.log.info('opt_in_alread() user: '******' has a mailHost ' + str(result['mailHost']))
if "gmx.pdx.edu" in result["mailHost"]:
self.log.info('opt_in_alread() user: '******' has a mailHost entry set to gmx.pdx.edu')
return True
return False
def populate(self):
"""Connects to Google domain, populates a list of usernames, and filters out against a static list of opt-outs and against the ldap directory. Creates a nested list of usernames."""
optouts = [ "janely", "leschins", "cfrl", "jensenmm", "polly", "nelsonk", "kerrigs", "pats", "wamserc", "wacke", "smithcc", "psu25042", "mackc", "powells", "mjantzen", "pcooper", "staplej", "pmueller", "ferguse" ]
if self.plevel == "prod":
gdomain = "pdx.edu"
elif self.plevel == "test":
gdomain = "gtest.pdx.edu"
elif self.plevel == "devl":
gdomain = "gdev.pdx.edu"
else:
raise Exception("Invalid plevel %s" % self.plevel)
guser = raw_input("Username: "******"Gathering all usernames for Google apps domain %s" % gdomain)
googleuserlists = google.allusernames()
optinuserlists = list()
print("Screening out early migration opt-opt users")
for userlist in googleuserlists:
optinuserlists.append([user for user in userlist if user not in optouts])
directory = psuldap()
directory.connect(ldapurl="ldap://ldap1.oit.pdx.edu")
self.userlists = list()
print("Screening out non-LDAP users")
for userlist in optinuserlists:
self.userlists.append([user for user in userlist if directory.exists("(uid=%s)" % user)])
print("Ready to launch!")
def is_oamed(self, login):
ldap = psuldap('/vol/certs')
ldap_host = self.prop.getProperty('ldap.read.host')
ldap_login = self.prop.getProperty('ldap.login')
ldap_password = self.prop.getProperty('ldap.password')
self.log.info('is_oamed(): connecting to LDAP: ' + ldap_host)
attr = 'eduPersonAffiliation'
ldap.connect( ldap_host, ldap_login, ldap_password)
res = ldap.search( searchfilter = 'uid=' + login, attrlist = [attr])
for (dn, result) in res:
if result.has_key(attr):
self.log.info('is_oamed() user: '******' has a ' + attr + ' of ' + str(result[attr]))
#print('is_oamed() user: '******' has a ' + attr + ' of ' + str(result[attr]))
for affiliation in result[attr]:
if affiliation in ['SPONSORED', 'SERVICE']:
self.log.info('is_oamed() user: '******' is not OAMed' )
#print('is_oamed() user: '******' is not OAMed' )
return False
return True
def imapsync(ldapuri=None, state_memcaches=None, nosync_memcaches=None, imapserver=None, adminuser=None, plevel="test", dryrun=True, runlimit=7200, user=None):
imapsync_dir = "/opt/google-imap/"
imapsync_cmd = imapsync_dir + "imapsync"
cyrus_pf = imapsync_dir + "cyrus.pf"
exclude_list = "'^Shared Folders|^mail/|^Junk$|^junk$|^JUNK$|^Spam$|^spam$|^SPAM$'"
whitespace_cleanup = " --regextrans2 's/[ ]+/ /g' --regextrans2 's/\s+$//g' --regextrans2 's/\s+(?=\/)//g' --regextrans2 's/^\s+//g' --regextrans2 's/(?=\/)\s+//g'"
folder_cases = " --regextrans2 's/^drafts$/[Gmail]\/Drafts/i' --regextrans2 's/^trash$/[Gmail]\/Trash/i' --regextrans2 's/^(sent|sent-mail)$/[Gmail]\/Sent Mail/i' --delete2foldersbutnot '^\[Gmail\]'"
extra_opts = " --delete2 --delete2folders --fast"
exitstatus = "premature"
if dryrun:
extra_opts = extra_opts + " --dry"
if runlimit <= 0:
raise Exception("Runlimit must be a positive integer.")
if plevel == "prod":
google_pf = imapsync_dir + "google-prod.pf"
google_domain = "pdx.edu"
elif plevel == "test":
google_pf = imapsync_dir + "google-test.pf"
google_domain = "gtest.pdx.edu"
else:
raise Exception("Plevel must be test or prod.")
command = imapsync_cmd + " --pidfile /tmp/imapsync-" + user + ".pid --host1 " + imapserver + " --port1 993 --user1 " + user + " --authuser1 " + adminuser + " --passfile1 " + cyrus_pf + " --host2 imap.gmail.com --port2 993 --user2 " + user + "@" + google_domain + " --passfile2 " + google_pf + " --ssl1 --ssl2 --maxsize 26214400 --authmech1 PLAIN --authmech2 XOAUTH -sep1 '/' --exclude " + exclude_list + folder_cases + whitespace_cleanup + extra_opts
cache = memcache.Client(servers=state_memcaches) # System state
nosync_cache = memcache.Client(servers=nosync_memcaches) # Users not-to-sync
cachekey = "(%s,auto)" % user
optinkey = "email_copy_progress.%s" % user
if nosync_cache.get(cachekey) != None: # If the key exists in this cache, skip the sync.
return (user, "nosync")
if cache.get(optinkey) != None: # If the user has opted in, skip the sync.
if nosync_cache.set(cachekey,{"status":"nosync"}) != True:
raise Exception("Could not set %s in nosync_cache." % cachekey)
return (user, "nosync")
directory = psuldap() # Our LDAP handle
directory.connect(ldapuri) # Anonymous bind
# Search for the user's mailHost attribute.
mailhostsearch = directory.search(searchfilter="(uid=%s)" % user, attrlist=["mailHost"])
# Loop through the search results. If any them match gmx.pdx.edu, set in the nosync cache.
for (dn, result) in mailhostsearch:
if result.has_key("mailHost"):
if "gmx.pdx.edu" in result["mailHost"]:
if nosync_cache.set(cachekey,{"status":"nosync"}) != True:
raise Exception("Could not set %s in nosync_cache." % cachekey)
return (user, "nosync")
# We can continue.
cachestate = cache.gets(cachekey)
if cachestate == None: # Maybe the cache has been cleared. Continue.
pass
# Well, it looks like the cache has another task's data for this user. Abort.
elif cachestate["status"] != "queued" or cachestate["taskid"] != imapsync.request.id:
raise Exception("Cache inconsistency error for user %s." % user)
# We're good to go. Let's set the cache with our new state.
runstate = {
"status":"running"
,"timestamp":int(time())
,"taskid":imapsync.request.id
,"worker":uname()[1]
}
cachelimit = runlimit + 10 # Fudge factor. 5 seconds for the sleep, 5 seconds for fudge.
if cache.cas(cachekey, runstate, time=cachelimit) != True: # Whoops, something changed. Abort.
raise Exception("Cache inconsistency error for user %s." % user)
syncprocess = subprocess.Popen(
args=shlex.split(command)
,bufsize=-1
,close_fds=True
,stdout=open("/dev/null",'w')
,stderr=None
)
starttime = time()
# While the process is running, and we're under the time limit
while (syncprocess.poll() == None) and ((time() - starttime) < runlimit):
sleep(5)
# Are we still running? Send a SIGTERM to the process and throw the task back on the queue.
# This is done to prevent one user from tying up a worker for longer than the runlimit.
if (syncprocess.poll() == None):
syncprocess.terminate() # Send SIGTERM
syncprocess.communicate() # Read stdin/out, and wait for process to terminate
exitstatus = "outtatime"
else:
if syncprocess.returncode == 0:
exitstatus = "ok"
else:
exitstatus = "error_%d" % syncprocess.returncode
cachestate = cache.gets(cachekey)
if cachestate == None: # Maybe the cache has been cleared. Continue.
pass
# Check to see if our cache state matches the previously set state.
if cachestate != runstate:
raise Exception("Cache inconsistency error for user %s." % user)
# We're good to go. Let's set the cache with our new state.
endstate = {
"status":"complete"
,"timestamp":int(time())
,"taskid":imapsync.request.id
,"worker":uname()[1]
,"returned":exitstatus
,"runtime":int(time() - starttime)
}
if cache.cas(cachekey, endstate) != True: # Whoops, something changed. Abort.
raise Exception("Cache inconsistency error for user %s." % user)
return (user, exitstatus)
