def update_mailRoutingAddress(self, login, deliveryAddr):
prop = Property( key_file = 'opt-in.key', properties_file = 'opt-in.properties')
self.log.info('update_mailRoutingAddress(): Updating mailRoutingAddress for user: '******'ldap.write.host')
ldap_login = prop.getProperty('ldap.login')
ldap_password = prop.getProperty('ldap.password')
cmd = '/usr/bin/ldapmodify -x -h ' + ldap_host + ' -D ' + ldap_login + " -w " + ldap_password
# Launch a Subprocess here to re-route email
input = '''
dn: uid=%s, ou=people, dc=pdx, dc=edu
changetype: modify
replace: mailRoutingAddress
mailRoutingAddress: %s@%s
''' % (login, login, deliveryAddr)
syncprocess = subprocess.Popen( shlex.split(cmd) ,stdin=subprocess.PIPE )
syncprocess.communicate(input)
while (syncprocess.poll() == None):
sleep(3)
self.log.info('update_mailRoutingAddress(): Continuing to update mailRoutingAddress for user: '******'update_mailRoutingAddress(): success for user: '******'update_mailRoutingAddress(): failed for user: ' + login)
return False
def route_to_google_old(self, login):
prop = Property( key_file = 'opt-in.key', properties_file = 'opt-in.properties')
self.log.info('route_to_google(): Routing mail to Google for user: '******'ldap.write.host')
ldap_login = prop.getProperty('ldap.login')
ldap_password = prop.getProperty('ldap.password')
cmd = '/usr/bin/ldapmodify -x -h ' + ldap_host + ' -D ' + ldap_login + " -w " + ldap_password
# Launch a Subprocess here to re-route email
input = '''
dn: uid=%s, ou=people, dc=pdx, dc=edu
changetype: modify
delete: mailHost
mailHost: cyrus.psumail.pdx.edu
-
add: mailHost
mailHost: gmx.pdx.edu
''' % login
syncprocess = subprocess.Popen( shlex.split(cmd) ,stdin=subprocess.PIPE )
syncprocess.communicate(input)
while (syncprocess.poll() == None):
sleep(3)
self.log.info('route_to_google(): continuing to route mail to Google for user: '******'route_to_google(): success for user: '******'route_to_google(): failed for user: ' + login)
return False
def is_web_suspended(self, login):
prop = Property( key_file = 'opt-in.key', properties_file = 'opt-in.properties')
web_suspended = prop.getProperty('web.suspended')
if web_suspended == 'True':
self.log.info('is_web_suspended(): user: ' + login + " visited while the opt-in web site was suspended")
return True
return False
def is_processing(self, login):
prop = Property( key_file = 'opt-in.key', properties_file = 'opt-in.properties')
memcache_url = prop.getProperty('memcache.url')
mc = memcache.Client([memcache_url], debug=0)
key = 'email_copy_progress.' + login
cached_data = mc.get(key)
if (cached_data == None):
return False
return True
def set_user(self, login, meta):
prop = Property( key_file = 'opt-in.key', properties_file = 'opt-in.properties')
memcache_url = prop.getProperty('memcache.url')
mc = memcache.Client([memcache_url], debug=0)
if self.META_IDENTITY in meta:
key = meta[self.META_IDENTITY]
mc.set(key, login)
self.log.info('psusys.PSUSys.set_user(), set user ' + login + ' in memcache')
else:
self.log.info('psusys.PSUSys.set_user(), failed to find: ' + self.META_IDENTITY + ' in META')
def route_to_psu_hack(self, login):
prop = Property( key_file = 'opt-in.key', properties_file = 'opt-in.properties')
memcache_url = prop.getProperty('memcache.url')
mc = memcache.Client([memcache_url], debug=0)
mc.set('psu_done.' + login, None)
mcq = MemcacheQueue('to_psu', mc)
mcq.add(login)
res = mc.get('psu_done.' + login)
while res == None:
res = mc.get('psu_done.' + login)
print 'Waiting for ' + login + ' to route to PSU'
sleep(10)
def copy_email_task_null(login):
prop = Property( key_file = 'opt-in.key', properties_file = 'opt-in.properties')
log = logging.getLogger()
memcache_url = prop.getProperty('memcache.url')
mc = memcache.Client([memcache_url], debug=0)
timer = 90;
key = 'email_copy_progress.' + login
while (timer <= 100):
mc.set(key, timer)
log.info('tasks.copy_email_task(), setting key: ' + key + ' to value: ' + str(timer) + " on: " + memcache_url)
timer = timer + 1
sleep(1)
return(True)
def copy_progress(self, login):
prop = Property( key_file = 'opt-in.key', properties_file = 'opt-in.properties')
memcache_url = prop.getProperty('memcache.url')
mc = memcache.Client([memcache_url], debug=0)
#cache_key = "copy_progress_%s" % (request.GET['login', 'default'])
key = 'email_copy_progress.' + login
cached_data = mc.get(key)
if (cached_data == None):
cached_data = 0
#data = simplejson.dumps(cached_data)
data = simplejson.dumps(cached_data)
self.log.info('PSUSys.copy_progress() called, memcache_url: ' + memcache_url + ", data: " + data + ' , login: ' + login)
return data
def get_user(self, meta):
prop = Property( key_file = 'opt-in.key', properties_file = 'opt-in.properties')
memcache_url = prop.getProperty('memcache.url')
mc = memcache.Client([memcache_url], debug=0)
login = None
if self.META_IDENTITY in meta:
key = meta[self.META_IDENTITY]
login = mc.get(key)
else:
self.log.info('psusys.PSUSys.get_user(), failed to find: ' + self.META_IDENTITY + ' in META')
if login == None:
login = '******'
self.log.info('psusys.PSUSys.get_user(), defaulting to user ' + login )
else:
self.log.info('psusys.PSUSys.get_user(), found user ' + login + ' in memcache')
return login
def setUp(self):
self.prop = Property()
self.host = self.prop.getProperty('destinyAgent.web_host')
self.username = self.prop.getProperty('destinyAgent.web_login')
self.password = self.prop.getProperty('destinyAgent.web_password')
self.log = logging.getLogger('ragve.destiny_agent.ceed')
self.log.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
hdlr = logging.FileHandler('/home/locngo/Documents/ktraks/myapp.log')
hdlr.setLevel(logging.DEBUG)
# create formatter
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
hdlr.setFormatter(formatter)
self.log.addHandler(ch)
self.log.addHandler(hdlr)
def is_allowed(self, login):
prop = Property( key_file = 'opt-in.key', properties_file = 'opt-in.properties')
# Does this user have an elligible account
if not self.is_oamed(login):
return False
# Is this user explicitly denied
deny_users = prop.getProperty('deny.users')
if login in deny_users:
return False
# Is this user explicitly allowed
allow_all = prop.getProperty('allow.all')
if allow_all == 'False':
allow_users = prop.getProperty('allow.users')
if login in allow_users:
return True
else:
return True
return False
def __init__(self):
self.MAX_MAIL_SIZE = pow(2,20) * 25
self.MAX_RETRY_COUNT = 5
self.prop = Property( key_file = 'opt-in.key', properties_file = 'opt-in.properties')
self.log = logging.getLogger('goblin.psusys')
print "Logging default handlers: " + str(self.log.handlers)
if len(self.log.handlers) == 0:
# No handlers for this logger, assume logging is not initialized..
logging.config.fileConfig('/vol/goblin/etc/logging.conf')
log = logging.getLogger('goblin.psusys')
self.setLogger(log)
self.META_IDENTITY = 'REMOTE_ADDR'
def setUp(self):
self.prop = Property()
self.host = self.prop.getProperty('destinyAgent.web_host')
self.username = self.prop.getProperty('destinyAgent.web_login')
self.password = self.prop.getProperty('destinyAgent.web_password')
self.log = logging.getLogger('ragve.destiny_agent.ceed')
self.log.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
hdlr = logging.FileHandler('/home/locngo/Documents/ktraks/myapp.log')
hdlr.setLevel(logging.DEBUG)
# create formatter
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
hdlr.setFormatter(formatter)
self.log.addHandler(ch)
self.log.addHandler(hdlr)
class Test(unittest.TestCase):
def setUp(self):
self.prop = Property()
self.host = self.prop.getProperty('destinyAgent.web_host')
self.username = self.prop.getProperty('destinyAgent.web_login')
self.password = self.prop.getProperty('destinyAgent.web_password')
self.log = logging.getLogger('ragve.destiny_agent.ceed')
self.log.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
hdlr = logging.FileHandler('/home/locngo/Documents/ktraks/myapp.log')
hdlr.setLevel(logging.DEBUG)
# create formatter
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
hdlr.setFormatter(formatter)
self.log.addHandler(ch)
self.log.addHandler(hdlr)
# def testLogin(self):
# da = DAgent(self.host,self.username,self.password)
# dwa = DestinyWebAgent(da.log,da.host)
# if dwa.login(da.username,da.password):
# self.assertTrue(dwa.selenium.is_element_present("//div[2]/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td[7]/a/img"))
# dwa.tearDown()
# self.assertTrue(True)
# else:
# self.assertTrue(dwa.selenium.sessionId==None)
# self.assertTrue(False)
#
# def testLogout(self):
# da = DAgent(self.host,self.username,self.password)
# dwa = DestinyWebAgent(da.log,da.host)
# if dwa.login(da.username,da.password):
# if(dwa.logout()):
# dwa.tearDown()
# self.assertTrue(True)
# else:
# self.assertTrue(False)
# else:
# self.assertTrue(True)
# def testBuildings(self):
# json_data=open('/home/locngo/Documents/ktraks/building.json')
# body = json.loads(json_data.read())
# json_data.close()
#
# blds = body["items"]
# da = DAgent(self.log,self.host,self.username,self.password)
#
# for key in blds.keys():
# bld = blds[key]
# da.log.info("Ready to add new building: " + key + " - " + str(bld))
# sg = Building(bld["lcode"],bld["name"],bld["addr"],bld["city"],bld["state"],bld["zip"],bld["campus"])
#
# createBuilding = da.createBuilding(sg)
# self.assertTrue(createBuilding)
# if(createBuilding):
# da.log.info("Successfully created building " + key)
# else:
# da.log.info("Couldn't create building " + key)
#
# if da.success == True:
# da.destiny_web_agent.tearDown()
# da.log.info("End of adding buildings")
# def testStudentGroups(self):
# json_data=open('/home/locngo/Documents/ktraks/group.json')
# body = json.loads(json_data.read())
# json_data.close()
#
# grs = body["items"]
# da = DAgent(self.log,self.host,self.username,self.password)
#
# for key in grs.keys():
# gr = grs[key]
# da.log.info("Ready to add new group: " + key + " - " + str(gr))
# sg = StudentGroup(gr["name"],gr["vendor_id"],gr["fed_tax_id"],gr["acode"],
# gr["addr1"],gr["addr2"],gr["city"],gr["state"],gr["zip"],
# gr["agency_phone_area"],gr["agency_phone_num"],gr["agency_phone_extn"],
# gr["agency_fax_area"],gr["agency_fax_num"],
# gr["person_phone_area"],gr["person_phone_num"],
# gr["person_fax_area"],gr["person_fax_num"],
# gr["person_email"],
# gr["person_name"],gr["www_home"],"",gr["reg_phrase"],gr["notes"])
#
# createGroup = da.createGroup(sg)
# self.assertTrue(createGroup)
# if(createGroup):
# da.log.info("Successfully created student group " + key)
# else:
# da.log.info("Couldn't create student group " + key)
#
# if da.success == True:
# da.destiny_web_agent.tearDown()
# da.log.info("End of adding groups")
# def testCreateNewInstructors(self):
# json_data=open('/home/locngo/Documents/ktraks/instructor.json')
# body = json.loads(json_data.read())
# json_data.close()
#
# insts = body["items"]
# da = DAgent(self.log,self.host,self.username,self.password)
#
# for key in insts.keys():
# inst = insts[key]
# da.log.info("Ready to add new instructor: " + key + " - " + str(inst))
# instructor = Instructor(inst["type"],inst["lastname"],inst["firstname"],inst["minitial"],
# inst["birthdate"],inst["active"],inst["psuid"],inst["tcode"],
# inst["employer"],inst["position"],inst["acode"],inst["home"]["street1"],inst["home"]["street2"],
# inst["home"]["city"],inst["home"]["state"],inst["home"]["country"],inst["home"]["zipcode"],
# inst["office"]["street1"],inst["office"]["street2"],inst["office"]["city"],
# inst["office"]["state"],inst["office"]["country"],inst["office"]["zipcode"],
# inst["home"]["phone_area"],inst["home"]["phone_number"],inst["home"]["fax_area"],inst["home"]["fax_number"],inst["office"]["phone_area"],inst["office"]["phone_number"],inst["office"]["fax_area"],inst["office"]["fax_number"],
# inst["home"]["email"],inst["office"]["email"],inst["bio"],inst["dist_qual"],
# inst["degree"],inst["www_home"],inst["miscinfo"])
#
# createInstructor = da.createInstructor(instructor)
# self.assertTrue(createInstructor)
# if(createInstructor):
# da.log.info("Successfully created instructor " + key)
# else:
# da.log.info("Couldn't create instructor " + key)
#
# if da.success == True:
# da.destiny_web_agent.tearDown()
# da.log.info("End of adding instructors")
def tearDown(self):
pass
# sel.click("//div[6]/div[2]/div/div/table/tbody/tr/td/form/table/tbody/tr[4]/td[2]/div[6]/input")
# time.sleep(1)
#multi-select Course Categories
sel.add_selection("name=courseCategoryStringArray", "4 Credit - CC0012")
sel.add_selection("name=courseCategoryStringArray", "Hybrid - CC0004")
time.sleep(1)
time.sleep(5)
if self.logout() == False:
return False
self.tearDown()
return True
if __name__ == '__main__':
prop = Property()
host = prop.getProperty('destinyAgent.web_host.test')
username = prop.getProperty('destinyAgent.web_login.test')
password = prop.getProperty('destinyAgent.web_password.test')
log = logging.getLogger('ragve.destiny_course_add.ceed')
log.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
hdlr = logging.FileHandler('/home/locngo/Documents/DestinyCourseSectionAddLog/destiny_course_add.log', mode='w')
hdlr.setLevel(logging.DEBUG)
# create formatter
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
def __init__(self):
self.fac_to_presync = []
self.prop = Property(key_file = 'opt-in.key', properties_file = 'opt-in.properties')
self.deny = self.prop.getProperty('deny.users')
class PreSync():
def __init__(self):
self.fac_to_presync = []
self.prop = Property(key_file = 'opt-in.key', properties_file = 'opt-in.properties')
self.deny = self.prop.getProperty('deny.users')
def gen_presync_test(self):
self.fac_to_presync = ['fogartym', 'bjms']
def gen_presync(self):
tmp_file = '/tmp/presync'
command = 'ldapsearch -x -LLL -h ldap-batch.oit.pdx.edu -w "jr83;gZ" -D uid=lms-system,ou=profile,dc=pdx,dc=edu -b ou=people,dc=pdx,dc=edu "(&(mailHost=cyrus.psumail.pdx.edu)(!(eduPersonAffiliation=DELETED))(!(psuUnixAccountStatus=pending))(!(eduPersonAffiliation=SERVICE))(!(eduPersonAffiliation=SPONSORED))(!(eduPersonAffiliation=TERMINATED)))" uid uniqueidentifier sn givenName eduPersonAffiliation'
syncprocess = subprocess.Popen(shlex.split(command), stdout=open(tmp_file, 'w'))
while (syncprocess.poll() == None):
sleep(30)
fh = open(tmp_file)
lines = fh.readlines()
haveRead = False
firstName = lastName = loginName = pidm = id = ""
role = ''
invalid = False
cmd = ''
facRe = re.compile('FAC', re.IGNORECASE)
staffRe = re.compile('STAFF', re.IGNORECASE)
empRe = re.compile('EMP', re.IGNORECASE)
stuRe = re.compile('STUDENT', re.IGNORECASE)
disabledRe = re.compile('DISABLED', re.IGNORECASE)
terminatedRe = re.compile('TERMINATED', re.IGNORECASE)
expiredRe = re.compile('EXPIRED', re.IGNORECASE)
for line in lines:
line = line.rstrip()
if (line):
try:
(a, v) = re.split(":", line)
except:
break
if (a == "uid"):
loginName = v.lstrip()
haveRead = True
if (a == "sn"):
lastName = v.lstrip()
if (a == "givenName"):
firstName = v.lstrip()
if (a == 'eduPersonAffiliation'):
if (facRe.search(v) is not None) or (staffRe.search(v) is not None) or (empRe.search(v) is not None) and not invalid:
role = 'Faculty'
elif (stuRe.search(v) is not None) and (role <> 'Faculty') and not invalid:
role = 'Student'
elif (disabledRe.search(v) is not None) or (terminatedRe.search(v) is not None) or (expiredRe.search(v) is not None):
invalid = True
role = ''
if (a == "uniqueidentifier"):
pidm = ""
id = v.lstrip()
if (id[0:1] == "B"):
id = "SPONSORED_" + id
else:
pidm = id[1:]
else:
if (haveRead == True) and (id):
if role == 'Faculty':
if loginName not in self.deny:
self.fac_to_presync.append(loginName)
elif role == 'Student':
pass
#print 'student ' + loginName
firstName = lastName = loginName = pidm = id = ""
role = ''
haveRead = False
invalid = False
random.shuffle(self.fac_to_presync)
def get_presync_list(self):
return self.fac_to_presync
def purge_queue(self):
rabbitmq_login = self.prop.getProperty('rabbitmq.login')
rabbitmq_pw = self.prop.getProperty('rabbitmq.password')
rabbitmq_host = self.prop.getProperty('rabbitmq.host')
credentials = pika.PlainCredentials(rabbitmq_login, rabbitmq_pw)
con = pika.BlockingConnection(pika.ConnectionParameters(host=rabbitmq_host, virtual_host=rabbitmq_login, credentials=credentials))
c = con.channel()
c.queue_purge( queue=rabbitmq_login)
def submit_task(self):
for login in self.fac_to_presync:
presync_email_task.apply_async(args=[login], queue='optinpresync')
def submit_test_task(self):
for login in self.fac_to_presync:
presync_email_test_task.apply_async(args=[login], queue='optinpresync')
def recover_copy_email_task(self, login):
'''
Recover from case where celery task has died unexpectantly. .. Don't do delete2
phase.
'''
prop = Property( key_file = 'opt-in.key', properties_file = 'opt-in.properties')
log = logging.getLogger('') # Logging is occuring within celery worker here
memcache_url = prop.getProperty('memcache.url')
mc = memcache.Client([memcache_url], debug=0)
psu_sys = PSUSys()
log.info("copy_email_task(): processing user: "******"copy_email_task(): has already completed opt-in: " + login)
# mc.set(key, 100)
#return(True)
mc.set(key, 40)
else:
log.info("copy_email_task(): has not already completed opt-in: " + login)
mc.set(key, 40)
# Enable Google email for the user
# This is the last item that the user should wait for.
psu_sys.enable_gmail(login)
mc.set(key, 50)
'''
# Synchronize email to Google (and wait)
log.info("copy_email_task(): first pass syncing email: " + login)
status = psu_sys.sync_email_delete2(login)
retry_count = 0
if (status == False) and (retry_count < self.MAX_RETRY_COUNT):
status = psu_sys.sync_email_delete2(login)
sleep(4 ** retry_count)
'''
mc.set(key, 60)
# Switch routing of email to flow to Google
log.info("copy_email_task(): Routing email to Google: " + login)
psu_sys.route_to_google(login)
mc.set(key, 70)
# Final email sync
log.info("copy_email_task(): second pass syncing email: " + login)
status = psu_sys.sync_email(login)
retry_count = 0
if (status == False) and (retry_count < self.MAX_RETRY_COUNT):
status = psu_sys.sync_email(login)
sleep(4 ** retry_count)
retry_count = retry_count + 1
mc.set(key, 80)
# The folowing items occur without the user waiting.
# Send conversion info email to users Google account
log.info("copy_email_task(): sending post conversion email to Google: " + login)
psu_sys.send_conversion_email_google(login)
# Send conversion info email to users PSU account
log.info("copy_email_task(): sending post conversion email to PSU: " + login)
psu_sys.send_conversion_email_psu(login)
mc.set(key, 100)
return(True)
def presync_email_task(self, login):
prop = Property( key_file = 'opt-in.key', properties_file = 'opt-in.properties')
log = logging.getLogger('') # Logging is occuring within celery worker here
memcache_url = prop.getProperty('memcache.url')
mc = memcache.Client([memcache_url], debug=0)
psu_sys = PSUSys()
max_process_time = 60 # Time is in minutes
log.info("presync_email_task(): processing user: "******"presync_email_task(): user currently opting-in: " + login)
return(True)
account_status = psu_sys.google_account_status(login)
# Check to make sure the user has a Google account
if account_status["exists"] == False:
log.info("presync_email_task(): user does not exist in Google: " + login)
return(True)
# Check for LDAP mail forwarding already (double checking), if
# already opt'd-in, then immediately return and mark as complete.
if (psu_sys.opt_in_already(login)):
log.info("presync_email_task(): has already completed opt-in: " + login)
return(True)
else:
log.info("presync_email_task(): has not already completed opt-in: " + login)
# We temporarily enable suspended accounts for the purposes of synchronization
if account_status["enabled"] == False:
log.info("presync_email_task(): temporarily enabling account: " + login)
psu_sys.enable_google_account(login) # Enable account if previously disabled
# Enable Google email for the user
log.info("presync_email_task(): temporarily enabling Google mail: " + login)
psu_sys.enable_gmail(login)
# Synchronize email to Google (and wait)
log.info("presync_email_task(): syncing email: " + login)
status = psu_sys.sync_email_delete2(login, max_process_time = max_process_time)
retry_count = 0
while (status == False) and (retry_count < self.MAX_RETRY_COUNT):
log.info("presync_email_task(): Retry syncing email: " + login)
status = psu_sys.sync_email_delete2(login, max_process_time = max_process_time)
sleep(4 ** retry_count)
retry_count = retry_count + 1
# Synchronization complete
# Disable Google email
log.info("presync_email_task(): disabling Google mail: " + login)
psu_sys.disable_gmail(login)
if account_status["enabled"] == False:
log.info("presync_email_task(): disabling account: " + login)
psu_sys.disable_google_account(login) # Enable account if previously disabled
# Call it good.
return(True)
def copy_email_task(self, login):
prop = Property( key_file = 'opt-in.key', properties_file = 'opt-in.properties')
log = logging.getLogger('') # Logging is occuring within celery worker here
memcache_url = prop.getProperty('memcache.url')
mc = memcache.Client([memcache_url], debug=0)
psu_sys = PSUSys()
log.info("copy_email_task(): processing user: "******"exists"] == False:
log.info("presync_email_task(): user does not exist in Google: " + login)
return(True)
# Check for LDAP mail forwarding already (double checking), if
# already opt'd-in, then immediately return and mark as complete.
if (psu_sys.opt_in_already(login)):
log.info("copy_email_task(): has already completed opt-in: " + login)
mc.set(key, 100)
return(True)
else:
log.info("copy_email_task(): has not already completed opt-in: " + login)
mc.set(key, 40)
# We temporarily enable suspended accounts for the purposes of synchronization
if account_status["enabled"] == False:
log.info("presync_email_task(): temporarily enabling account: " + login)
psu_sys.enable_google_account(login) # Enable account if previously disabled
mc.set(key, 45)
# Send conversion info email to users Google account
log.info("copy_email_task(): conversion in progress email: " + login)
psu_sys.send_conversion_email_in_progress(login)
# Enable Google email for the user
# This is the last item that the user should wait for.
psu_sys.enable_gmail(login)
mc.set(key, 50)
# Synchronize email to Google (and wait)
log.info("copy_email_task(): first pass syncing email: " + login)
status = psu_sys.sync_email_delete2(login)
retry_count = 0
while (status == False) and (retry_count < self.MAX_RETRY_COUNT):
log.info("copy_email_task(): Retry of first pass syncing email: " + login)
status = psu_sys.sync_email_delete2(login)
sleep(4 ** retry_count)
retry_count = retry_count + 1
mc.set(key, 60)
# Switch routing of email to flow to Google
log.info("copy_email_task(): Routing email to Google: " + login)
psu_sys.route_to_google(login)
mc.set(key, 70)
# Final email sync
log.info("copy_email_task(): second pass syncing email: " + login)
status = psu_sys.sync_email(login)
retry_count = 0
while (status == False) and (retry_count < self.MAX_RETRY_COUNT):
log.info("copy_email_task(): Retry of second pass syncing email: " + login)
status = psu_sys.sync_email(login)
sleep(4 ** retry_count)
retry_count = retry_count + 1
mc.set(key, 80)
# The folowing items occur without the user waiting.
# Send conversion info email to users Google account
log.info("copy_email_task(): sending post conversion email to Google: " + login)
psu_sys.send_conversion_email_google(login)
# Send conversion info email to users PSU account
log.info("copy_email_task(): sending post conversion email to PSU: " + login)
psu_sys.send_conversion_email_psu(login)
# If the account was disabled, well...
if account_status["enabled"] == False:
log.info("presync_email_task(): disabling account: " + login)
psu_sys.disable_google_account(login) # Enable account if previously disabled
mc.set(key, 90)
mc.set(key, 100)
return(True)
class Test(unittest.TestCase):
def setUp(self):
self.prop = Property()
self.host = self.prop.getProperty('destinyAgent.web_host')
self.username = self.prop.getProperty('destinyAgent.web_login')
self.password = self.prop.getProperty('destinyAgent.web_password')
self.log = logging.getLogger('ragve.destiny_agent.ceed')
self.log.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
hdlr = logging.FileHandler('/home/locngo/Documents/ktraks/myapp.log')
hdlr.setLevel(logging.DEBUG)
# create formatter
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
hdlr.setFormatter(formatter)
self.log.addHandler(ch)
self.log.addHandler(hdlr)
# def testLogin(self):
# da = DAgent(self.host,self.username,self.password)
# dwa = DestinyWebAgent(da.log,da.host)
# if dwa.login(da.username,da.password):
# self.assertTrue(dwa.selenium.is_element_present("//div[2]/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td[7]/a/img"))
# dwa.tearDown()
# self.assertTrue(True)
# else:
# self.assertTrue(dwa.selenium.sessionId==None)
# self.assertTrue(False)
#
# def testLogout(self):
# da = DAgent(self.host,self.username,self.password)
# dwa = DestinyWebAgent(da.log,da.host)
# if dwa.login(da.username,da.password):
# if(dwa.logout()):
# dwa.tearDown()
# self.assertTrue(True)
# else:
# self.assertTrue(False)
# else:
# self.assertTrue(True)
# def testBuildings(self):
# json_data=open('/home/locngo/Documents/ktraks/building.json')
# body = json.loads(json_data.read())
# json_data.close()
#
# blds = body["items"]
# da = DAgent(self.log,self.host,self.username,self.password)
#
# for key in blds.keys():
# bld = blds[key]
# da.log.info("Ready to add new building: " + key + " - " + str(bld))
# sg = Building(bld["lcode"],bld["name"],bld["addr"],bld["city"],bld["state"],bld["zip"],bld["campus"])
#
# createBuilding = da.createBuilding(sg)
# self.assertTrue(createBuilding)
# if(createBuilding):
# da.log.info("Successfully created building " + key)
# else:
# da.log.info("Couldn't create building " + key)
#
# if da.success == True:
# da.destiny_web_agent.tearDown()
# da.log.info("End of adding buildings")
# def testStudentGroups(self):
# json_data=open('/home/locngo/Documents/ktraks/group.json')
# body = json.loads(json_data.read())
# json_data.close()
#
# grs = body["items"]
# da = DAgent(self.log,self.host,self.username,self.password)
#
# for key in grs.keys():
# gr = grs[key]
# da.log.info("Ready to add new group: " + key + " - " + str(gr))
# sg = StudentGroup(gr["name"],gr["vendor_id"],gr["fed_tax_id"],gr["acode"],
# gr["addr1"],gr["addr2"],gr["city"],gr["state"],gr["zip"],
# gr["agency_phone_area"],gr["agency_phone_num"],gr["agency_phone_extn"],
# gr["agency_fax_area"],gr["agency_fax_num"],
# gr["person_phone_area"],gr["person_phone_num"],
# gr["person_fax_area"],gr["person_fax_num"],
# gr["person_email"],
# gr["person_name"],gr["www_home"],"",gr["reg_phrase"],gr["notes"])
#
# createGroup = da.createGroup(sg)
# self.assertTrue(createGroup)
# if(createGroup):
# da.log.info("Successfully created student group " + key)
# else:
# da.log.info("Couldn't create student group " + key)
#
# if da.success == True:
# da.destiny_web_agent.tearDown()
# da.log.info("End of adding groups")
# def testCreateNewInstructors(self):
# json_data=open('/home/locngo/Documents/ktraks/instructor.json')
# body = json.loads(json_data.read())
# json_data.close()
#
# insts = body["items"]
# da = DAgent(self.log,self.host,self.username,self.password)
#
# for key in insts.keys():
# inst = insts[key]
# da.log.info("Ready to add new instructor: " + key + " - " + str(inst))
# instructor = Instructor(inst["type"],inst["lastname"],inst["firstname"],inst["minitial"],
# inst["birthdate"],inst["active"],inst["psuid"],inst["tcode"],
# inst["employer"],inst["position"],inst["acode"],inst["home"]["street1"],inst["home"]["street2"],
# inst["home"]["city"],inst["home"]["state"],inst["home"]["country"],inst["home"]["zipcode"],
# inst["office"]["street1"],inst["office"]["street2"],inst["office"]["city"],
# inst["office"]["state"],inst["office"]["country"],inst["office"]["zipcode"],
# inst["home"]["phone_area"],inst["home"]["phone_number"],inst["home"]["fax_area"],inst["home"]["fax_number"],inst["office"]["phone_area"],inst["office"]["phone_number"],inst["office"]["fax_area"],inst["office"]["fax_number"],
# inst["home"]["email"],inst["office"]["email"],inst["bio"],inst["dist_qual"],
# inst["degree"],inst["www_home"],inst["miscinfo"])
#
# createInstructor = da.createInstructor(instructor)
# self.assertTrue(createInstructor)
# if(createInstructor):
# da.log.info("Successfully created instructor " + key)
# else:
# da.log.info("Couldn't create instructor " + key)
#
# if da.success == True:
# da.destiny_web_agent.tearDown()
# da.log.info("End of adding instructors")
def tearDown(self):
pass
class PSUSys:
def __init__(self):
self.MAX_MAIL_SIZE = pow(2,20) * 25
self.MAX_RETRY_COUNT = 5
self.prop = Property( key_file = 'opt-in.key', properties_file = 'opt-in.properties')
self.log = logging.getLogger('goblin.psusys')
print "Logging default handlers: " + str(self.log.handlers)
if len(self.log.handlers) == 0:
# No handlers for this logger, assume logging is not initialized..
logging.config.fileConfig('/vol/goblin/etc/logging.conf')
log = logging.getLogger('goblin.psusys')
self.setLogger(log)
self.META_IDENTITY = 'REMOTE_ADDR'
def setLogger(self, logger):
self.log = logger
def large_emails(self, login):
imap_host = self.prop.getProperty('imap.host')
imap_login = self.prop.getProperty('imap.login')
imap_password = self.prop.getProperty('imap.password')
self.log.info('PSUSys.large_emails() login: '******'PSUSys.large_emails() imapstat host: ' + imap_host)
stat = ims.stat(login)
msg_list = ims.bigmessages(login, stat['mbox_list'], self.MAX_MAIL_SIZE )
large_emails = []
for folder in msg_list.keys():
for msg in msg_list[folder]:
large_email = {}
large_emails.append(large_email)
for key in ['Subject', 'Date', 'From']:
if key in msg:
large_email[key] = msg[key]
else:
large_email[key] = 'none'
return large_emails
def opt_in_already(self, login):
ldap = psuldap('/vol/certs')
ldap_host = self.prop.getProperty('ldap.read.host')
ldap_login = self.prop.getProperty('ldap.login')
ldap_password = self.prop.getProperty('ldap.password')
self.log.info('opt_in_alread(): connecting to LDAP: ' + ldap_host)
ldap.connect( ldap_host, ldap_login, ldap_password)
res = ldap.search( searchfilter = 'uid=' + login, attrlist = ['mailHost'])
for (dn, result) in res:
if result.has_key("mailHost"):
self.log.info('opt_in_alread() user: '******' has a mailHost ' + str(result['mailHost']))
if "gmx.pdx.edu" in result["mailHost"]:
self.log.info('opt_in_alread() user: '******' has a mailHost entry set to gmx.pdx.edu')
return True
return False
def is_oamed(self, login):
ldap = psuldap('/vol/certs')
ldap_host = self.prop.getProperty('ldap.read.host')
ldap_login = self.prop.getProperty('ldap.login')
ldap_password = self.prop.getProperty('ldap.password')
self.log.info('is_oamed(): connecting to LDAP: ' + ldap_host)
attr = 'eduPersonAffiliation'
ldap.connect( ldap_host, ldap_login, ldap_password)
res = ldap.search( searchfilter = 'uid=' + login, attrlist = [attr])
for (dn, result) in res:
if result.has_key(attr):
self.log.info('is_oamed() user: '******' has a ' + attr + ' of ' + str(result[attr]))
#print('is_oamed() user: '******' has a ' + attr + ' of ' + str(result[attr]))
for affiliation in result[attr]:
if affiliation in ['SPONSORED', 'SERVICE']:
self.log.info('is_oamed() user: '******' is not OAMed' )
#print('is_oamed() user: '******' is not OAMed' )
return False
return True
def get_ldap_attr(self, login, attr):
ldap = psuldap('/vol/certs')
ldap_host = self.prop.getProperty('ldap.read.host')
ldap_login = self.prop.getProperty('ldap.login')
ldap_password = self.prop.getProperty('ldap.password')
self.log.info('opt_in_alread(): connecting to LDAP: ' + ldap_host)
ldap.connect( ldap_host, ldap_login, ldap_password)
res = ldap.search( searchfilter = 'uid=' + login, attrlist = ['mailHost'])
for (dn, result) in res:
if result.has_key(attr):
return str(result[attr])
return None
def route_to_google_null(self, login):
self.log.info('route_to_google(): routing mail to google for user: '******'opt-in.key', properties_file = 'opt-in.properties')
# Does this user have an elligible account
if not self.is_oamed(login):
return False
# Is this user explicitly denied
deny_users = prop.getProperty('deny.users')
if login in deny_users:
return False
# Is this user explicitly allowed
allow_all = prop.getProperty('allow.all')
if allow_all == 'False':
allow_users = prop.getProperty('allow.users')
if login in allow_users:
return True
else:
return True
return False
# Temporary hack till Adrian sorts-out the access issues for modifying LDAP
def route_to_google_hack(self, login):
prop = Property( key_file = 'opt-in.key', properties_file = 'opt-in.properties')
memcache_url = prop.getProperty('memcache.url')
mc = memcache.Client([memcache_url], debug=0)
mc.set('gmx_done.' + login, None)
mcq = MemcacheQueue('to_google', mc)
mcq.add(login)
res = mc.get('gmx_done.' + login)
while res == None:
res = mc.get('gmx_done.' + login)
print 'Waiting for ' + login + ' to route to Google'
sleep(10)
def route_to_google_needswork(self, login):
self.update_mailHost(login, 'gmx.pdx.edu')
self.update_mailRoutingAddress(login, )
def route_to_google_old(self, login):
prop = Property( key_file = 'opt-in.key', properties_file = 'opt-in.properties')
self.log.info('route_to_google(): Routing mail to Google for user: '******'ldap.write.host')
ldap_login = prop.getProperty('ldap.login')
ldap_password = prop.getProperty('ldap.password')
cmd = '/usr/bin/ldapmodify -x -h ' + ldap_host + ' -D ' + ldap_login + " -w " + ldap_password
# Launch a Subprocess here to re-route email
input = '''
dn: uid=%s, ou=people, dc=pdx, dc=edu
changetype: modify
delete: mailHost
mailHost: cyrus.psumail.pdx.edu
-
add: mailHost
mailHost: gmx.pdx.edu
''' % login
syncprocess = subprocess.Popen( shlex.split(cmd) ,stdin=subprocess.PIPE )
syncprocess.communicate(input)
while (syncprocess.poll() == None):
sleep(3)
self.log.info('route_to_google(): continuing to route mail to Google for user: '******'route_to_google(): success for user: '******'route_to_google(): failed for user: '******'cyrus.psumail.pdx.edu')
self.update_mailRoutingAddress(login, 'odin.pdx.edu')
def route_to_google(self, login):
status = self.update_mailHost(login, 'gmx.pdx.edu')
retry_count = 0
while (status == False) and (retry_count < self.MAX_RETRY_COUNT):
self.log.error("route_to_google(): mailHost: Retrying LDAP update")
status = self.update_mailHost(login, 'gmx.pdx.edu')
retry_count = retry_count + 1
status = self.update_mailRoutingAddress(login, 'pdx.edu')
retry_count = 0
while (status == False) and (retry_count < self.MAX_RETRY_COUNT):
self.log.error("route_to_google(): mailRoutingAddress: Retrying LDAP update")
status = self.update_mailRoutingAddress(login, 'pdx.edu')
retry_count = retry_count + 1
def update_mailHost(self, login, deliveryHost):
prop = Property( key_file = 'opt-in.key', properties_file = 'opt-in.properties')
self.log.info('update_mailHost(): Routing mail to psu for user: '******'ldap.write.host')
ldap_login = prop.getProperty('ldap.login')
ldap_password = prop.getProperty('ldap.password')
cmd = '/usr/bin/ldapmodify -x -h ' + ldap_host + ' -D ' + ldap_login + " -w " + ldap_password
# Launch a Subprocess here to re-route email
input = '''
dn: uid=%s, ou=people, dc=pdx, dc=edu
changetype: modify
replace: mailHost
mailHost: %s
''' % (login, deliveryHost)
syncprocess = subprocess.Popen( shlex.split(cmd), stdin=subprocess.PIPE )
syncprocess.communicate(input)
while (syncprocess.poll() == None):
sleep(3)
self.log.info('update_mailHost(): continuing to route mail to psu for user: '******'update_mailHost(): success for user: '******'update_mailHost(): failed for user: '******'opt-in.key', properties_file = 'opt-in.properties')
self.log.info('update_mailRoutingAddress(): Updating mailRoutingAddress for user: '******'ldap.write.host')
ldap_login = prop.getProperty('ldap.login')
ldap_password = prop.getProperty('ldap.password')
cmd = '/usr/bin/ldapmodify -x -h ' + ldap_host + ' -D ' + ldap_login + " -w " + ldap_password
# Launch a Subprocess here to re-route email
input = '''
dn: uid=%s, ou=people, dc=pdx, dc=edu
changetype: modify
replace: mailRoutingAddress
mailRoutingAddress: %s@%s
''' % (login, login, deliveryAddr)
syncprocess = subprocess.Popen( shlex.split(cmd) ,stdin=subprocess.PIPE )
syncprocess.communicate(input)
while (syncprocess.poll() == None):
sleep(3)
self.log.info('update_mailRoutingAddress(): Continuing to update mailRoutingAddress for user: '******'update_mailRoutingAddress(): success for user: '******'update_mailRoutingAddress(): failed for user: '******'opt-in.key', properties_file = 'opt-in.properties')
memcache_url = prop.getProperty('memcache.url')
mc = memcache.Client([memcache_url], debug=0)
mc.set('psu_done.' + login, None)
mcq = MemcacheQueue('to_psu', mc)
mcq.add(login)
res = mc.get('psu_done.' + login)
while res == None:
res = mc.get('psu_done.' + login)
print 'Waiting for ' + login + ' to route to PSU'
sleep(10)
def route_to_google_future(self, login):
self.log.info('route_to_google(): routing mail to google for user: '******'/vol/certs')
# ldapsearch -x -h ldap.oit.pdx.edu -b 'dc=pdx, dc=edu' uid=dennis mailhost
ldap_host = self.prop.getProperty('ldap.host')
ldap_login = self.prop.getProperty('ldap.login')
ldap_password = self.prop.getProperty('ldap.password')
#self.log.info('opt_in_alread(): connecting to LDAP: ' + ldap_host)
ldap.connect( ldap_host, ldap_login, ldap_password)
dn = 'uid=' + login + ',ou=people,dc=pdx,dc=edu'
ldap.mod_attribute(dn, 'mailHost', 'gmx.pdx.edu')
def route_to_psu_future(self, login):
ldap = psuldap('/vol/certs')
# ldapsearch -x -h ldap.oit.pdx.edu -b 'dc=pdx, dc=edu' uid=dennis mailhost
ldap_host = self.prop.getProperty('ldap.host')
ldap_login = self.prop.getProperty('ldap.login')
ldap_password = self.prop.getProperty('ldap.password')
#self.log.info('opt_in_alread(): connecting to LDAP: ' + ldap_host)
ldap.connect( ldap_host, ldap_login, ldap_password)
dn = 'uid=' + login + ',ou=people,dc=pdx,dc=edu'
ldap.mod_attribute(dn, 'mailHost', 'cyrus.psumail.pdx.edu')
def get_user(self, meta):
prop = Property( key_file = 'opt-in.key', properties_file = 'opt-in.properties')
memcache_url = prop.getProperty('memcache.url')
mc = memcache.Client([memcache_url], debug=0)
login = None
if self.META_IDENTITY in meta:
key = meta[self.META_IDENTITY]
login = mc.get(key)
else:
self.log.info('psusys.PSUSys.get_user(), failed to find: ' + self.META_IDENTITY + ' in META')
if login == None:
login = '******'
self.log.info('psusys.PSUSys.get_user(), defaulting to user ' + login )
else:
self.log.info('psusys.PSUSys.get_user(), found user ' + login + ' in memcache')
return login
def set_user(self, login, meta):
prop = Property( key_file = 'opt-in.key', properties_file = 'opt-in.properties')
memcache_url = prop.getProperty('memcache.url')
mc = memcache.Client([memcache_url], debug=0)
if self.META_IDENTITY in meta:
key = meta[self.META_IDENTITY]
mc.set(key, login)
self.log.info('psusys.PSUSys.set_user(), set user ' + login + ' in memcache')
else:
self.log.info('psusys.PSUSys.set_user(), failed to find: ' + self.META_IDENTITY + ' in META')
def send_conversion_email_null(self, login):
addr = login + '@pdx.edu'
self.log.info('send_conversion_email(): sending mail to user: '******'send_conversion_email_in_progress(): sending mail to user: '******'/vol/goblin/src/conversion_email_in_progress ' + login
syncprocess = subprocess.Popen( shlex.split(cmd) )
while (syncprocess.poll() == None):
sleep(3)
self.log.info('send_conversion_email_in_progress(): continuing to send mail for user: '******'send_conversion_email_in_progress(): success for user: '******'send_conversion_email_in_progress(): failed for user: '******'send_conversion_email_psu(): sending mail to user: '******'/vol/goblin/src/conversion_email_psu ' + login
syncprocess = subprocess.Popen( shlex.split(cmd) )
while (syncprocess.poll() == None):
sleep(3)
self.log.info('send_conversion_email_psu(): continuing to send mail for user: '******'send_conversion_email_psu(): success for user: '******'send_conversion_email_psu(): failed for user: '******'send_conversion_email_google(): sending mail to user: '******'/vol/goblin/src/conversion_email_google ' + login
syncprocess = subprocess.Popen( shlex.split(cmd) )
while (syncprocess.poll() == None):
sleep(3)
self.log.info('send_conversion_email_google(): continuing to send mail for user: '******'send_conversion_email_google(): success for user: '******'send_conversion_email_google(): failed for user: '******'enable_gail(): Enabling gmail for user: '******'is_gmail_enabled(): Checking if gmail is enabled for user: '******'google.email')
domain = self.prop.getProperty('google.domain')
pw = self.prop.getProperty('google.password')
client = gdata.apps.organization.service.OrganizationService(email=email, domain=domain, password=pw)
retry_count = 0; status = False; result = False
while (status == False) and (retry_count < self.MAX_RETRY_COUNT):
try:
client.ProgrammaticLogin()
customerId = client.RetrieveCustomerId()["customerId"]
userEmail = login + '@pdx.edu'
result = (client.RetrieveOrgUser( customerId, userEmail )['orgUnitPath'] == 'people')
status = True
except CaptchaRequired :
self.log.error('is_gmail_enabled(): Captcha being requested')
sleep(1)
except BadAuthentication :
self.log.error('is_gmail_enabled(): Authentication Error' )
sleep(1)
except Exception, e :
self.log.error('is_gmail_enabled(): Exception occured: ' + str(e))
sleep(1)
# Retry if not an obvious non-retryable error
retry_count = retry_count + 1
return result
