def get_resources():
global screens_active
login_label.text = 'Retrieving Data'
# Find and access folder we are uploading to
globalVars.resources = Initialize.findFolder(new_user_setup)
# Find file in the App folder and get its contents
Initialize.findFile()
# Remove all login screen widgets to prepare to display the main screen widgets
f.remove_widget(pb)
f.remove_widget(user_field)
f.remove_widget(pass_field)
f.remove_widget(login_button)
f.remove_widget(login_label)
# Prompt user for session id
screens_active[0] = False
screens_active[1] = True
f.add_widget(sesh_id_label)
f.add_widget(sesh_id_field)
sesh_id_field.focus = True
def test_click_into_set_userinfo_page(self):
try:
print 'start test_click_into_set_userinfo_page test ... '
Initialize.init_case(self) #处理开屏广告是否存在的情况
InitLogin.init_login(self) #如果没有登陆则登陆秒拍
sleep(5)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/bottom_my').click() #点击底导上的我进入我的页面
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/user_info_layout').click(
) #点击整条用户信息进入个人页面
sleep(4)
self.driver.find_element_by_id('com.yixia.videoeditor:id/text_info'
).click() #点击用户名和用户描述进入编辑个人资料页面
sleep(1)
self.driver.find_element_by_id('com.yixia.videoeditor:id/titleLeft'
).click() #编辑个人资料页面点击返回到个人主页
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/edit_icon').click() #点击右箭头进入编辑个人资料页面
sleep(1)
self.driver.press_keycode('4') #击返回按钮回到个人主页
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/titleLeft').click() #点击返回到我的页面
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/bottom_feed').click() #点击回到热门页面
sleep(2)
except Exception, e:
print traceback.format_exc()
CutScreenshot.cutScreenShot(self, sys._getframe().f_code.co_name)
def test_setting_page_logOut(self):
try:
print 'start test_setting_page_logOut test ... '
Initialize.init_case(self) #处理开屏广告是否存在的情况
InitLogin.init_login(self) #如果没有登陆则登陆秒拍
sleep(5)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/bottom_my').click() #点击底导的我
sleep(2)
self.driver.swipe(200, 1000, 200, 100, 1000) #我的页面向上滑动
sleep(1)
self.driver.swipe(200, 1000, 200, 100, 1000) #我的页面向上滑动
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/setting_layout').click() #我的页面点击设置
sleep(2)
self.driver.swipe(200, 800, 200, 200, 1000) #设置页面向上滑动
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/logout_button').click() #点击退出登录按钮
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/dialog_right_buton').click() #点击确定
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/titleLeft').click() #点击返回到我的页面
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/bottom_feed').click() #点击回到首页首页
sleep(1)
except Exception, e:
print traceback.format_exc()
CutScreenshot.cutScreenShot(self, sys._getframe().f_code.co_name)
def test_mypage_topinfo_click(self):
try:
print 'start test_mypage_topinfo_click test ... '
Initialize.init_case(self) #处理开屏广告是否存在的情况
InitLogin.init_login(self) #如果没有登陆则登陆秒拍
sleep(5)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/bottom_my').click() #点击底导上的我进入我的页面
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/icon_header').click() #点击个人信息上面的头像
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/imageview').click() #点击关闭头像
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/user_info_layout').click(
) #点击整条用户信息进入个人页面
sleep(2)
self.driver.press_keycode('4') #点击返回按钮返回到我的页面
sleep(2)
self.driver.find_element_by_name('申请认证').click() #点击申请认证
sleep(2)
self.driver.press_keycode('4') #点击返回按钮返回到我的页面
sleep(3)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/videos_layout').click(
) #点击整条视频缩略图区域进入我的个人主页
sleep(2)
self.driver.press_keycode('4') #点击返回按钮返回到我的页面
sleep(2)
except Exception, e:
print traceback.format_exc()
CutScreenshot.cutScreenShot(self, sys._getframe().f_code.co_name)
def test_version_check(self):
try:
print 'start test_version_check test ... '
Initialize.init_case(self) #处理开屏广告是否存在的情况
InitLogin.init_login(self) #如果没有登陆则登陆秒拍
sleep(5)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/bottom_my').click() #点击底导的我
sleep(2)
self.driver.swipe(200, 1000, 200, 100, 1000) #我的页面向上滑动
sleep(1)
self.driver.swipe(200, 1000, 200, 100, 1000) #我的页面向上滑动
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/setting_layout').click() #我的页面点击设置
sleep(2)
#self.driver.find_element_by_name('版本检测').click() #点击版本检测选项
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/check_version').click() #点击版本检测选项
sleep(3)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/titleLeft').click() #点击返回按钮
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/bottom_feed').click() #点击首页回到首页
sleep(1)
except Exception, e:
print traceback.format_exc()
CutScreenshot.cutScreenShot(self, sys._getframe().f_code.co_name)
def __init__(self):
# Explaining super is out of the scope of this article
# So please google it if you're not familar with it
# Simple reason why we use it here is that it allows us to
# access variables, methods etc in the design.py file
super(self.__class__, self).__init__()
# self.communicate
self.setupUi(self) # This is defined in design.py file automatically
# It sets up layout and widgets that are defined
self.dirbrowser.clicked.connect(self.browser_directory)
self.gdsbrowser.clicked.connect(lambda: self.browser_gdsfile())
self.techbrowser.clicked.connect(lambda: self.browser_techfile())
self.actionLoadSession.triggered.connect(self.LoadSession_file)
self.actionSaveSession.triggered.connect(self._savession)
self.celllist.clicked.connect(self.getDesign)
self.run.clicked.connect(self.getRUN)
self.cellname.textChanged.connect(self.setGDSText)
self.rundir.textChanged.connect(self.setGDSText)
Initialize.init(self, ".pwrrc")
currenttime = '{0:%Y%m%d%H%M}'.format(datetime.datetime.now())
self.date.setText(currenttime)
def test_banner(self):
try:
print 'start test_banner test ... '
Initialize.init_case(self) #处理开屏广告是否存在的情况
InitLogin.init_login(self) #如果没有登陆则登陆秒拍
sleep(7)
#查找banner翻页元素获取banner总页数
eles=self.driver.find_elements_by_xpath('//android.widget.LinearLayout[contains(@id,com.yixia.videoeditor:id/banner_dots_layout)]/android.widget.ImageView')
l=len(eles)*2
print l
#获取banner元素
banner=self.driver.find_element_by_id('com.yixia.videoeditor:id/banner_img')
#向左滑动banner总数*2次
width=self.driver.get_window_size().get('width')
height=self.driver.get_window_size().get('height')
#print width
#print height
x_start=540*width/720
x_end=160*width/720
y=450*height/1280
for i in range(0,l):
self.driver.swipe(x_start, y, x_end, y)
sleep(2)
#向左滑动banner总数*2次
for i in range(0,l):
self.driver.swipe(160, 450,546, 409)
sleep(2)
sleep(5)
#点击任一banner
banner.click()
self.driver.keyevent('4')
sleep(5)
except Exception,e:
print traceback.format_exc()
CutScreenshot.cutScreenShot(self,sys._getframe().f_code.co_name)
def drag_to_change_category_order(self):
try:
'''首页平道分类测试
1、改变首页分类的顺序
'''
print 'start drag_to_change_category_order test ... '
Initialize.init_case(self)
sleep(5)
self.driver.find_element_by_id('com.yixia.videoeditor:id/more').click() #点击展开频道更多
sleep(1)
ele = self.driver.find_elements_by_id('com.yixia.videoeditor:id/icon')
self.driver.drag_and_drop(ele[6],ele[8])
sleep(2)
self.driver.drag_and_drop(ele[4],ele[2])
sleep(2)
self.driver.drag_and_drop(ele[8],ele[2])
sleep(2)
self.driver.drag_and_drop(ele[4],ele[10])
sleep(2)
self.driver.drag_and_drop(ele[10],ele[1])
sleep(2)
self.driver.find_element_by_xpath('//android.widget.FrameLayout/android.widget.FrameLayout/android.widget.FrameLayout/android.widget.LinearLayout/android.widget.LinearLayout/android.widget.RelativeLayout/android.widget.ImageView').click() #点击关闭全部平道面板
sleep(2)
self.driver.find_element_by_id('com.yixia.videoeditor:id/more').click() #点击展开频道更多
sleep(2)
except Exception,e:
print traceback.format_exc()
CutScreenshot.cutScreenShot(self,sys._getframe().f_code.co_name)
def main():
# 读取配置参数
conf = Initialize.Conf()
conf.load(r'data\config.json')
# 读取模块分工文件
md = Initialize.MD()
md.load(conf.module_define)
# 解析命令行参数
parser = MyArgParse()
parser.parse_args()
if not parser.is_valid(): # 命令行参数非法
error_message = parser.command_error()
print(error_message)
elif parser.is_client_side(): # 客户端模式
conf.client_mode = True
client_job(conf, md)
else: # 服务端模式
# 利用命令行参数，更新配置参数
conf = parser.update_config(conf)
conf.client_mode = False
print(conf.to_str())
# 读取无效记录文件
inva = Initialize.Invalid()
inva.load(conf.invalid_define)
inva.print()
conf.invalid_records = inva.invalid_list
server_job(conf, md)
def switch_Opencategory_at_shouye_by_click(self):
try:
'''首页平道分类测试
1、点击切换完首页展开的频道分类
'''
print 'start switch_Opencategory_at_shouye_by_click test ... '
Initialize.init_case(self)
sleep(5)
self.driver.find_element_by_id('com.yixia.videoeditor:id/more').click() #点击展开频道更多
sleep(1)
ele = self.driver.find_elements_by_id('com.yixia.videoeditor:id/icon')
for i in range(0,14):
ele[i].click()
sleep(1)
self.driver.find_element_by_id('com.yixia.videoeditor:id/more').click() #点击展开频道更多
sleep(1)
self.driver.swipe(500, 900, 500, 300) #向上滑动
ele = self.driver.find_elements_by_id('com.yixia.videoeditor:id/icon')
ele[len(ele) - 1].click()
sleep(1)
self.driver.find_element_by_id('com.yixia.videoeditor:id/more').click() #点击展开频道更多
sleep(1)
ele[len(ele) - 2].click()
sleep(1)
self.driver.find_element_by_id('com.yixia.videoeditor:id/more').click() #点击展开频道更多
sleep(1)
self.driver.find_element_by_xpath('//android.widget.FrameLayout/android.widget.FrameLayout/android.widget.FrameLayout/android.widget.LinearLayout/android.widget.LinearLayout/android.widget.RelativeLayout/android.widget.ImageView').click() #点击关闭全部平道面板
sleep(2)
self.driver.find_element_by_id('com.yixia.videoeditor:id/more').click() #点击展开频道更多
sleep(2)
except Exception,e:
print traceback.format_exc()
CutScreenshot.cutScreenShot(self,sys._getframe().f_code.co_name)
def switch_category_at_shouye_by_click(self):
try:
'''首页平道分类测试
1、点击切换完首页所有的分类
'''
print 'start switch_category_at_shouye_by_click test ... '
Initialize.init_case(self)
sleep(5)
ele1 = self.driver.find_elements_by_xpath('//android.widget.FrameLayout/android.widget.LinearLayout/android.widget.FrameLayout/android.widget.RelativeLayout/android.widget.FrameLayout/android.widget.LinearLayout/android.widget.RelativeLayout/android.widget.HorizontalScrollView/android.widget.RelativeLayout/android.widget.LinearLayout/android.widget.RadioButton') #获取首页所有的二级分类
ele1[0].click()
sleep(2)
ele1[1].click()
sleep(2)
ele1[2].click()
sleep(2)
for i in range(0,12):
ele1[3].click()
sleep(2)
ele1[4].click()
sleep(2)
ele1[5].click()
sleep(2)
self.driver.find_element_by_id('com.yixia.videoeditor:id/more').click() #点击展开频道更多
sleep(2)
self.driver.find_element_by_xpath('//android.widget.FrameLayout/android.widget.FrameLayout/android.widget.FrameLayout/android.widget.LinearLayout/android.widget.LinearLayout/android.widget.RelativeLayout/android.widget.ImageView').click() #点击关闭全部平道面板
sleep(2)
self.driver.find_element_by_id('com.yixia.videoeditor:id/more').click() #点击展开频道更多
sleep(2)
except Exception,e:
print traceback.format_exc()
CutScreenshot.cutScreenShot(self,sys._getframe().f_code.co_name)
def LoadSession_file(self):
dialog = QtGui.QFileDialog(self)
dialog.setFileMode(QtGui.QFileDialog.AnyFile)
dialog.setViewMode(QtGui.QFileDialog.Detail)
if dialog.exec_():
fileNames = dialog.selectedFiles()
file = fileNames.pop(0)
Initialize.init(self, file)
def goIntoPage(self):
Initialize.init_case(self)
sleep(3)
#判断当前Activity是否为被测试Activity，如果不是抛出异常，让用例执行失败
if (Common.isTestActivity(self)):
print "被测试的Activity为：" + self.driver.current_activity + ",是被测试的Activity"
else:
print "当前Activity为：" + self.driver.current_activity
Common.excuteFailed("不是被测试activity")
def main():
# configuration
conf = Initialize.Conf()
conf.load(r'data\config.json')
# module define
md = Initialize.MD()
md.load(conf.module_define)
fetch_analyze(conf, md)
def main (k, m="means", init_type="random"):
# Starting clustering timer
start_cluster = timeit.default_timer()
# Initialize clusters
if init_type == "random":
initial_clusters = Initialize.random_centers(k)
else:
init_type = "kplusplus"
initial_clusters = Initialize.kmeans_plusplus(k, train_images_flat,\
dist_fn=Distance.sumsq)
# Run clustering algorithm
final_responsibilities, final_clusters = Kmeans.kmeans(k,train_images_flat,
initial_clusters, distfn = Distance.sumsq, method=m)
# Find and print clustering time
end_cluster = timeit.default_timer()
clustering_time = end_cluster - start_cluster
print "Time spent clustering : ", clustering_time
# Save representative images to file.
title = m + "_" + init_type + "_cluster" + str(k)
File.save_images(k, train_images, final_responsibilities,
final_clusters, title)
###########################################################################
# Calculate Accuracy #
###########################################################################
# Calculate final accuracy for clusters
final, cluster_set = Accuracy.final_accuracy(final_responsibilities,
train_labels, train_images_flat, final_clusters)
# Now see how well we can classify the dataset
start_cluster_test = timeit.default_timer()
predictions = ClassifyClusters.classify(cluster_set, test_images_flat,
test_labels, distfn = Distance.sumsq)
finish_cluster_test = timeit.default_timer()
# find time it took to test
testing_time = finish_cluster_test - start_cluster_test
print "Time spent testing : ", testing_time
###########################################################################
# Outputs #
###########################################################################
# k, prediction level, cluster_set,
results = {"k" : k, "prediction_accuracy" : predictions[1],
"cluster_means" : cluster_set, "cluster_stats" : final,
"clustering_time" : clustering_time, "testing_time" : testing_time}
with open('./results/' + title + '/' + title + '_results.json', 'w') as outfile:
json.dump(results, outfile, cls=File.NumpyEncoder)
def goIntoPage(self):
Initialize.init_case(self)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/bottom_my').click()
sleep(2)
#判断当前Activity是否为被测试Activity，如果不是抛出异常，让用例执行失败
if (Common.isTestActivity(self)):
print "被测试的Activity为：" + self.driver.current_activity + ",是被测试的Activity"
else:
Common.excuteFailed("不是被测试activity")
def initialize(config="", inputRealImage=True):
"""
Initialize smul with the given configuration file.
config: Name of file to read configuration from.
(need not be provided to processes other
than the root process)
"""
SMPI.init()
rank = SMPI.rank()
Initialize.initialize(config, inputRealImage=inputRealImage)
def test_message_items_all_click(self):
try:
print 'start test_message_items_all_click test ... '
Initialize.init_case(self) #处理开屏广告是否存在的情况
InitLogin.init_login(self) #如果没有登陆则登陆秒拍
sleep(5)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/bottom_my').click() #点击底导上的我进入我的页面
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/message_layout').click() #点击进入消息通知页面
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/rl_newfriend').click() #点击进入新的好友消息页面
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/titleLeft').click() #点击返回到消息页面
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/rl_like').click() #点击进入赞页面
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/titleLeft').click() #点击返回到消息页面
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/rl_comment').click() #点击进入评论消息页面
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/titleLeft').click() #点击返回到消息页面
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/rl_at').click() #点击进入@我的消息页面
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/titleLeft').click() #点击返回到消息页面
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/rl_notification').click(
) #点击进入系统消息页面
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/titleLeft').click() #点击返回到消息页面
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/titleLeft').click() #点击返回到我的页面
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/bottom_feed').click() #点击回到热门页面
sleep(2)
except Exception, e:
print traceback.format_exc()
CutScreenshot.cutScreenShot(self, sys._getframe().f_code.co_name)
def test_clear_cache(self):
try:
print 'start test_clear_cache test ... '
Initialize.init_case(self) #处理开屏广告是否存在的情况
InitLogin.init_login(self) #如果没有登陆则登陆秒拍
sleep(5)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/bottom_my').click() #点击底导的我
sleep(2)
self.driver.swipe(200, 1000, 200, 100, 1000) #我的页面向上滑动
sleep(1)
self.driver.swipe(200, 1000, 200, 100, 1000) #我的页面向上滑动
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/setting_layout').click() #我的页面点击设置
sleep(2)
self.driver.swipe(200, 1000, 200, 100, 1000) #设置页面向上滑动
sleep(1)
#self.driver.find_element_by_name('清空缓存').click() #点击清空缓存
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/clear_cache').click() #点击清空缓存
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/dialog_left_buton').click(
) #点击否取消清空缓存
self.el = None
try:
self.el = self.driver.find_element_by_name('0')
except Exception, ex:
#print Exception,':',ex
pass
self.assertIsNone(self.el) #通过缓存不为0来判断缓存没有清空
sleep(2)
#self.driver.find_element_by_name('清空缓存').click() #点击清空缓存
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/clear_cache').click() #点击清空缓存
sleep(5)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/dialog_right_buton').click(
) #点击是清空缓存
sleep(1)
self.el = None
self.txt = None
try:
#self.el = self.driver.find_element_by_name('0')
self.txt = self.driver.find_elements_by_id(
'com.yixia.videoeditor:id/cache_num'
)[1].text #获取缓存的值(index0是系统版本号)
print self.txt.decode("ascii").encode("utf-8")
except Exception, ex:
print Exception, ':', ex
pass
def test_comment_etc(self):
try:
print 'start test_comment_etc test ... '
Initialize.init_case(self) #处理开屏广告是否存在的情况
InitLogin.init_login(self) #如果没有登陆则登陆秒拍
sleep(5)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/bottom_my').click() #点击底导的我
sleep(2)
self.driver.swipe(200, 1000, 200, 100, 1000) #我的页面向上滑动
sleep(1)
self.driver.swipe(200, 1000, 200, 100, 1000) #我的页面向上滑动
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/setting_layout').click() #我的页面点击设置
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/message_tips_text').click() #点击新消息提醒
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/message_fans').click() #打开新粉丝push
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/message_fans').click() #关闭新粉丝push
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/message_comment').click() #打开评论push
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/message_comment').click() #关闭评论push
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/message_good').click() #打开赞push
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/message_good').click() #关闭赞push
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/titleLeft').click() #点击返回按钮退出该页面
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/titleLeft').click() #点击返回按钮
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/bottom_feed').click() #点击首页回到首页
sleep(1)
except Exception, e:
print traceback.format_exc()
CutScreenshot.cutScreenShot(self, sys._getframe().f_code.co_name)
def GetData():
try:
InitialParameters = Initialize.getParam()
StagingPath = InitialParameters[2]
APIView = InitialParameters[3].split(',')
APICategory = InitialParameters[4].split(',')
BaseURl = InitialParameters[5].replace("'", "")
type_param = 'csv'
row_param = 'rows'
RequestURLList = []
WriteToLog('\nFetching Files form http://data.consumerfinance.gov using API')
WriteToLog('------------------------------------------------------------------------------------------------------------------')
for i in range(0, len(APIView)):
# print(APICategory[i].replace("'",""),'\t\t',APIView[i].replace("'",""))
RequestURLList.append(BaseURl + APIView[i].replace("'", "") + '/' + row_param + '.' + type_param)
#print(RequestURLList[i])
URLData = request.urlopen(RequestURLList[i])
CSVRaw = URLData.read()
CSVData = str(CSVRaw).strip("b'")
lines = CSVData.split("\\n")
CSVFileName = StagingPath + APICategory[i].replace("'", "") + '.' + type_param
CSVFile = open(CSVFileName, 'w')
WriteToLog('Loading : '+APICategory[i].replace("'", "") + '.' + type_param+' <-- '+ RequestURLList[i])
#print(CSVFileName)
for line in lines:
CSVFile.write(line + "\n")
CSVFile.close()
WriteToLog('------------------------------------------------------------------------------------------------------------------')
except Exception as e:
FailureRevert(3,e)
def WriteToLog(msg):
InitialParameters = Initialize.getParam()
LogFileName = InitialParameters[14]
LogFile = open(LogFileName, 'a')
LogFile.write(msg+'\n')
LogFile.close()
print(msg)
def updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel = None):
if pComponent["componenttype"] == Cassette.componentType:
return Cassette.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == Add.componentType:
return Add.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == Evaporate.componentType:
return Evaporate.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == Transfer.componentType:
return Transfer.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == React.componentType:
return React.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == Prompt.componentType:
return Prompt.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == Install.componentType:
return Install.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == Comment.componentType:
return Comment.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == TrapF18.componentType:
return TrapF18.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == EluteF18.componentType:
return EluteF18.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == Initialize.componentType:
return Initialize.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == Mix.componentType:
return Mix.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == Move.componentType:
return Move.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == Summary.componentType:
return Summary.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == ExternalAdd.componentType:
return ExternalAdd.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
else:
raise Exception("Unknown component type: " + pComponent["componenttype"])
def DataUpdateFlag():
# check if the file is present if not then create
InitialParameters = Initialize.getParam()
LoadFileName = InitialParameters[15]
LogFileName = InitialParameters[14]
MainLogTable = InitialParameters[18]
if os.path.isfile(LoadFileName) == True:
print('Update Already in Progress !!!')
exit()
else:
LoadStartDTTM=str(datetime.datetime.now()).split('.')[0]
LoadFile = open(LoadFileName, 'w')
LoadFile.close()
LogFile = open(LogFileName, 'w')
LogFile.close()
WriteToLog('Update Process Initiated . . . '+time.strftime('%d %b %Y %X'))
LoadIndex=FirebirdDB.GetLoadIndex(MainLogTable)
LoadIndex+=1
LoadEndDTTM=str(datetime.datetime.now()).split('.')[0]
global STEP_COUNT
FirebirdDB.UpdateMainLog(MainLogTable,LoadIndex,STEP_COUNT,'Pre-Pre',LogFileName.split('\\')[-1],'Created Log File to Capture Update Progress Status',0,LoadStartDTTM,LoadEndDTTM)
STEP_COUNT+=1
FirebirdDB.UpdateMainLog(MainLogTable,LoadIndex,STEP_COUNT,'Pre-Pre',LoadFileName.split('\\')[-1],'Created Trig File to Indicate \'Update in Progress\'',0,LoadStartDTTM,LoadEndDTTM)
STEP_COUNT+=1
def PreLoadTasks():
InitialParameters = Initialize.getParam()
LoadStartDTTM=str(datetime.datetime.now()).split('.')[0]
TableName = InitialParameters[10]
CloneTableName=InitialParameters[12]
ViewName=InitialParameters[16]
MainLogTable = InitialParameters[18]
global STEP_COUNT
# Clone Main Table
try:
LoadIndex=FirebirdDB.GetLoadIndex(MainLogTable)
FirebirdDB.CloneTable(TableName,CloneTableName)
TableCount=FirebirdDB.GetTableCount(CloneTableName)
LoadEndDTTM=str(datetime.datetime.now()).split('.')[0]
except Exception as e:
FailureRevert(1,e)
else:
WriteToLog('\nClone Table ('+CloneTableName.upper()+') Created.')
FirebirdDB.UpdateMainLog(MainLogTable,LoadIndex,STEP_COUNT,'Pre-Pre',CloneTableName,'Clone Table Created',TableCount,LoadStartDTTM,LoadEndDTTM)
STEP_COUNT+=1
#Swap Views
try:
FirebirdDB.ViewSwap(CloneTableName,ViewName)
ViewCount=FirebirdDB.GetTableCount(ViewName)
LoadEndDTTM=str(datetime.datetime.now()).split('.')[0]
except Exception as e2:
FailureRevert(2,e2)
else:
WriteToLog('View Swap ('+ViewName.upper()+' --> '+CloneTableName.upper()+') Completed.')
FirebirdDB.UpdateMainLog(MainLogTable,LoadIndex,STEP_COUNT,'Pre-Pre',ViewName,'Swap Main view to Clone Table',ViewCount,LoadStartDTTM,LoadEndDTTM)
STEP_COUNT+=1
def test_into_setting_page(self):
try:
print 'start test_into_setting_page test ... '
'''按钮的点击测试
1、分别对静音、wifi下自动播放、3G较底画质开关、静音进行了打开和关闭的点击测试
'''
Initialize.init_case(self) #处理开屏广告是否存在的情况
InitLogin.init_login(self) #如果没有登陆则登陆秒拍
sleep(5)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/bottom_my').click() #点击底导的我
sleep(2)
self.driver.swipe(200, 1000, 200, 100, 1000) #我的页面向上滑动
sleep(1)
self.driver.swipe(200, 1000, 200, 100, 1000) #我的页面向上滑动
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/setting_layout').click() #我的页面点击设置
sleep(2)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/wifi_setting').click() #打开wifi自动播放
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/wifi_setting').click() #关闭wifi自动播放
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/upload_3g_setting').click() #打开3G拍摄
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/upload_3g_setting').click() #关闭3G拍摄
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/mute_setting').click() #打开静音
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/mute_setting').click() #关闭静音
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/titleLeft').click() #点击返回按钮
sleep(1)
self.driver.find_element_by_id(
'com.yixia.videoeditor:id/bottom_feed').click() #点击首页回到首页
sleep(1)
except Exception, e:
print traceback.format_exc()
CutScreenshot.cutScreenShot(self, sys._getframe().f_code.co_name)
def switch_category_at_shouye_by_slide(self): try: '''切换分类页面 1、滑动切换完首页所有的分类 ''' print 'start switch_category_at_shouye_by_slide test ... ' Initialize.init_case(self) sleep(5) for i in range(0,20): self.driver.swipe(700, 900, 50, 900) #向右滑动 sleep(2) for i in range(0,20): self.driver.swipe(50, 900, 700, 900) #向左滑动 sleep(2) except Exception,e: print traceback.format_exc() CutScreenshot.cutScreenShot(self,sys._getframe().f_code.co_name)
def connectSocket(self):
if not self.isConnectionHealthy():
if (self.__gui.getOauthStr() and self.__gui.getNameStr()
and self.__gui.getChnlStr()):
self.__socket = Socket.openSocket(
str(self.__gui.getOauthStr()),
str(self.__gui.getNameStr()), str(self.__gui.getChnlStr()))
self.isConnected(Initialize.joinRoom(self.__socket), True)
def connectSocket(self):
if not self.isConnectionHealthy():
if (self.__gui.getOauthStr() and self.__gui.getNameStr() and self.__gui.getChnlStr()):
self.__socket = Socket_local.Socket_local()
self.__socket.openSocket(str(self.__gui.getOauthStr()), str(self.__gui.getNameStr()), str(self.__gui.getChnlStr()))
connectionError = Initialize.joinRoom(self.__socket, self.__gui.getNameStr())
if connectionError:
self.__socket = ''
self.isConnected(connectionError == 0, True, connectionError)
def cross_validation(NetFile, NetName, fold=10):
auc_val = np.zeros(12)
methods = [
'CN', 'Salton', 'Jaccard', 'Sorenson', 'HPI', 'HDI', 'LHN-I', 'PA',
'AA', 'RA', 'LP', 'Katz'
]
for i in range(10):
MatrixAdjacency_Net, MaxNodeNum = Initialize.Init(NetFile)
MatrixAdjacency_Train, MatrixAdjacency_Test = Initialize.Divide(
NetFile, MatrixAdjacency_Net, MaxNodeNum, NetName)
auc = get_auc_all(MatrixAdjacency_Train, MatrixAdjacency_Test,
MaxNodeNum)
auc = np.array(auc)
auc_val += auc
auc_val = (0.1) * auc_val
auc_val = pd.Series(auc_val, index=methods)
auc_val.name = 'auc'
return auc_val
def cross_validation(NetFile, NetName, fold=10):
methods = [
'CN', 'Salton', 'Jaccard', 'Sorenson', 'HPI', 'HDI', 'LHN-I', 'PA',
'AA', 'RA', 'LP', 'Katz'
]
auc_mix = np.zeros(23)
for i in range(10):
MatrixAdjacency_Net, MaxNodeNum = Initialize.Init(NetFile)
MatrixAdjacency_Train, MatrixAdjacency_Test = Initialize.Divide(
NetFile, MatrixAdjacency_Net, MaxNodeNum, NetName)
auc = get_auc_mix(MatrixAdjacency_Train, MatrixAdjacency_Test,
MaxNodeNum)
auc = np.array(auc)
auc_mix += auc
auc_mix = (0.1) * auc_mix
auc_mix = pd.Series(auc_mix, index=methods + (list(range(1, 12))))
auc_mix.name = 'auc'
return auc_mix
def click_shouye_rebang_faxian_wo(self):
try:
'''按钮的点击测试
1、分别对底导上的首页、热榜、发现，我进行了点击
'''
print 'start click_shouye_rebang_faxian_wo test ... '
Initialize.init_case(self)
sleep(5)
self.driver.find_element_by_id('com.yixia.videoeditor:id/bottom_feed').click() #点击底导上的首页
sleep(2)
self.driver.find_element_by_id('com.yixia.videoeditor:id/bottom_friend').click() #点击底导上的热榜
sleep(2)
self.driver.find_element_by_id('com.yixia.videoeditor:id/bottom_message_tip').click() #点击底导上的发现
sleep(2)
self.driver.find_element_by_id('com.yixia.videoeditor:id/bottom_my').click() #点击底导的我
sleep(2)
self.driver.find_element_by_id('com.yixia.videoeditor:id/bottom_feed').click() #点击底导上的首页
sleep(2)
except Exception,e:
print traceback.format_exc()
CutScreenshot.cutScreenShot(self,sys._getframe().f_code.co_name)
def createFromComponent(nSequenceID,
pComponent,
username,
database,
systemModel=None):
if pComponent["componenttype"] == Cassette.componentType:
return Cassette.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == Add.componentType:
return Add.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == Evaporate.componentType:
return Evaporate.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == Transfer.componentType:
return Transfer.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == React.componentType:
return React.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == Prompt.componentType:
return Prompt.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == Install.componentType:
return Install.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == Comment.componentType:
return Comment.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == TrapF18.componentType:
return TrapF18.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == EluteF18.componentType:
return EluteF18.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == Initialize.componentType:
return Initialize.createFromComponent(nSequenceID, pComponent,
username, database, systemModel)
elif pComponent["componenttype"] == Mix.componentType:
return Mix.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == Move.componentType:
return Move.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == Summary.componentType:
return Summary.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == ExternalAdd.componentType:
return ExternalAdd.createFromComponent(nSequenceID, pComponent,
username, database, systemModel)
else:
raise Exception("Unknown component type: " +
pComponent["componenttype"])
def DisplayLogContent():
#global LogValue
CurrentLogValue=''
InitialParameters = Initialize.getParam()
LogFileName = InitialParameters[14]
try:
LogFile = open(LogFileName, 'r')
except Exception as e:
#print('Log File Error : ',e)
CurrentLogValue='Unable to Display Log.Please Wait . . .'
else:
CurrentLogValue = LogFile.read()
LogContentDisplay.config(text=CurrentLogValue)
LogContentDisplay.after(200,DisplayLogContent)
def FailureRevert(Step,Error):
global FAIL_FLAG,STEP_COUNT
ErrorString={1:'Creating Clone Table',
2:'Swapping View to Clone Table',
3:'Fetching Data from Online API',
4:'Creating Reject File',
5:'Truncating Staging Table',
6:'Loading Data into Staging Table',
7:'Appending Data to Main Table',
8:'Swapping View to Main Table',
9:'Archiving Reject File',
10:'Deleting Trig File',
11:'Archiving Log File'}
WriteToLog('Error During : '+str(ErrorString.get(Step)))
WriteToLog('Description :'+str(Error))
InitialParameters = Initialize.getParam()
MainLogTable = InitialParameters[18]
TableName = InitialParameters[10]
CloneTableName=InitialParameters[12]
LoadStartDTTM=str(datetime.datetime.now()).split('.')[0]
LoadEndDTTM=str(datetime.datetime.now()).split('.')[0]
LoadIndex=FirebirdDB.GetLoadIndex(MainLogTable)
FirebirdDB.UpdateMainLog(MainLogTable,LoadIndex,STEP_COUNT,'Error',str(ErrorString.get(Step)),str(Error),0,LoadStartDTTM,LoadEndDTTM)
STEP_COUNT+=1
#Revert Tasks if needed
if Step==1:
RevertList=['T','L','F']
elif Step in (3,4,5,6):
RevertList=['S','T','L','F']
elif Step == 7:
FirebirdDB.CloneTable(CloneTableName,TableName)
RevertList=['S','T','L','F']
else:
messagebox.showinfo("Alert !!!", "Error in Swapping View.Please check Logs!!!\n\n\nTerminating Upload Process.")
exit()
#fun the post function
PostLoadTasks(*RevertList)
def MainTableAppend():
try:
InitialParameters = Initialize.getParam()
MainLogTable = InitialParameters[18]
MainLoadStartDTTM=str(datetime.datetime.now()).split('.')[0]
StagTableName = InitialParameters[11]
TableName = InitialParameters[10]
WriteToLog('\nAdding new Records to Main Table ('+TableName.upper()+')')
WriteToLog('----------------------------------------------------------------------------------------------------------')
MainRowCount=FirebirdDB.GetMainRecordCount(TableName,StagTableName)
FirebirdDB.UpdateMainTable(TableName,StagTableName)
WriteToLog('Loading : '+TableName.upper()+'<-- '+StagTableName.upper()+' [Count('+str(MainRowCount)+')]')
WriteToLog('----------------------------------------------------------------------------------------------------------')
#Update the Main Log
LoadIndex=FirebirdDB.GetLoadIndex(MainLogTable)
TableCount=FirebirdDB.GetTableCount(TableName)
MainLoadEndDTTM=str(datetime.datetime.now()).split('.')[0]
except Exception as e:
FailureRevert(7,e)
else:
global STEP_COUNT
FirebirdDB.UpdateMainLog(MainLogTable,LoadIndex,STEP_COUNT,'Data-Load',TableName,'Main Table Loaded',TableCount,MainLoadStartDTTM,MainLoadEndDTTM)
STEP_COUNT+=1
import fdb
import Initialize
InitialParameters=Initialize.getParam()
DBLocation=InitialParameters[1]
DBName=InitialParameters[6]
UserName=InitialParameters[7]
Password=InitialParameters[8]
DBPath=DBLocation+DBName
CreateComment="create database '"+str(DBPath)+"' user '"+str(UserName)+"' password '"+str(Password)+"'"
print('DatabasePath :',DBPath)
con = fdb.create_database(CreateComment)
print("Database (",DBName,") created Sucessfully .")
con = fdb.connect(database=DBPath, user=UserName, password=Password)
cur = con.cursor()
cur.execute("recreate table consumercomplaint (DateReceived date,Product varchar(100),SubProduct varchar(100),Issue varchar(100),SubIssue varchar(100),ConsumerComplaint varchar(5000),CompanyPublicResponse varchar(5000),Company varchar(100),State varchar(5),ZIPCode varchar(10),SubmittedVia varchar(20),DateSentCompany date,CompanyResponseConsumer varchar(100),TimelyResponseSts varchar(5),ConsumerDisputedSts varchar(5),ComplaintID int)")
con.commit()
print("Table ConsumerComplaint created Sucessfully .")
cur.execute("create unique index unique_ComplaintID on consumercomplaint(ComplaintID)")
con.commit()
print("Uniqie Index on ConsumerComplaint(ComplaintID)created Sucessfully .")
cur.execute("recreate table consumercomplaint_staging (DateReceived varchar(100),Product varchar(100),SubProduct varchar(100),Issue varchar(100),SubIssue varchar(100),ConsumerComplaint varchar(5000),CompanyPublicResponse varchar(5000),Company varchar(100),State varchar(50),ZIPCode varchar(100),SubmittedVia varchar(100),DateSentCompany varchar(100),CompanyResponseConsumer varchar(100),TimelyResponseSts varchar(10),ConsumerDisputedSts varchar(10),ComplaintID int)")
con.commit()
print("Table consumercomplaint_staging created Sucessfully .")
cur.execute("recreate view ConsumerComplaintView as select DateReceived,Product,SubProduct,Issue,SubIssue,ConsumerComplaint,CompanyPublicResponse,Company,State,ZIPCode,SubmittedVia,DateSentCompany,CompanyResponseConsumer,TimelyResponseSts,ConsumerDisputedSts,ComplaintID from consumercomplaint")
con.commit()
print("View ConsumerComplaintView created Sucessfully .")
cur.execute("recreate table consumercomplaint_copy (DateReceived date,Product varchar(100),SubProduct varchar(100),Issue varchar(100),SubIssue varchar(100),ConsumerComplaint varchar(5000),CompanyPublicResponse varchar(5000),Company varchar(100),State varchar(5),ZIPCode varchar(10),SubmittedVia varchar(20),DateSentCompany date,CompanyResponseConsumer varchar(100),TimelyResponseSts varchar(5),ConsumerDisputedSts varchar(5),ComplaintID int)")
con.commit()
def ViewLog(root, ImageFrame, ImageLabel, photo, MainPageFrame,ViewLogButton,FetchDataButton):
#Disable the Button and destroy the Image
ViewLogButton.config(state=DISABLED)
FetchDataButton.config(state=DISABLED)
ImageLabel.destroy()
#Create Top and Bottom Frame
TopViewFrame=Frame(ImageFrame)
TopViewFrame.pack(side=TOP)
BottopViewFrame=Frame(ImageFrame)
BottopViewFrame.pack(side=BOTTOM)
#Label to Display Log with Scroll Bar
#Function for Scroll Bar
def ScrollLogFunc(event):
ViewCanvas.configure(scrollregion=ViewCanvas.bbox("all"),height=370,width=800)
#canvas to Implement Scroll Bar for Log
ViewCanvas=Canvas(TopViewFrame)
CanvasFrame=Frame(ViewCanvas)
ScrollLogY=Scrollbar(TopViewFrame, orient="vertical",command=ViewCanvas.yview)
ViewCanvas.configure(yscrollcommand=ScrollLogY.set)
ViewCanvas.pack(side=LEFT)
ScrollLogY.pack(side=RIGHT,fill=Y)
ViewCanvas.create_window((0,0),window=CanvasFrame,anchor=NW)
#Label to display the Log Details
ViewLogDisplay=Label(CanvasFrame,text='Please Select a Log Category.',anchor=NW,justify=LEFT,wraplength=700)
CanvasFrame.bind("<Configure>",ScrollLogFunc)
ViewLogDisplay.pack(side=BOTTOM)
#DropDown Button For Archive Log
var = StringVar()
var.set('')
InitialParameters=Initialize.getParam()
LogPath=InitialParameters[19]
#LogPath=r'C:\Users\Priyank\Desktop\MIS\Scripting Languages\DataAnalyst-Project(Python)\Coding\Logs'
LogList=[]
for filename in os.listdir(LogPath):
#if filename.split('.')[-1]=='log' and filename.split('_')[0]=='DataUpdateLog':
LogList.append(filename)
def func(value):
LogFile=LogPath +'\\' + str(value)
Log = open(LogFile, 'r')
LogValue = Log.read()
ViewLogDisplay.config(text=LogValue)
ChoiceLabel.config(text='Archived Process Logs -')
ChoiceLabel=Label(CanvasFrame,text='LOGS !!!')
ChoiceLabel.pack(side=LEFT)
ChoiceDropDown = OptionMenu(CanvasFrame, var, *LogList, command=func)
ChoiceDropDown.pack(side=RIGHT)
#Function to Display the Logs based on selection
#APi and Source file info from Log table
def APILog(ChoiceLabel):
ChoiceLabel.config(text='Source File and API Logs -')
var.set('')
APLDataFrame=FirebirdDB.DisplayAPILog()
APLDataFrame['FileLen']=APLDataFrame.FileName.map(len)
MaxLength=0
for i in range(0,len(APLDataFrame)):
if APLDataFrame.loc[i,'FileLen'] >MaxLength:
MaxLength=APLDataFrame.loc[i,'FileLen']
for i in range(0,len(APLDataFrame)):
APLDataFrame.loc[i,'FileName']=str(APLDataFrame.loc[i,'FileName']).ljust(MaxLength,'_')
m,s=divmod(APLDataFrame.loc[i,'Duration'],60)
h,m=divmod(m,60)
DurationTime=str(int(h)).rjust(2,'0')+':'+str(int(m)).rjust(2,'0')+':'+str(int(s)).rjust(2,'0')
APLDataFrame.loc[i,'Duration']=DurationTime
#print(APLDataFrame.loc[i,'LoadStartDTTM'])
APLDataFrame.loc[i,'LoadStartDTTM']=datetime.datetime.strptime(APLDataFrame.loc[i,'LoadStartDTTM'], "%Y-%m-%d %H:%M:%S.%f").strftime("%d %b %y %H:%M:%S")
APLDataFrame.loc[i,'LoadStartDTTM']=str(APLDataFrame.loc[i,'LoadStartDTTM']).ljust(30,' ').upper()
#print(datetime.datetime.strptime(APLDataFrame.loc[i,'LoadStartDTTM']),"%d/%m/%Y").strftime("%d%b%y"))
APLDataFrame.loc[i,'LoadIndex']=str(int(APLDataFrame.loc[i,'LoadIndex']))
APLDataFrame.loc[i,'LoadOrder']=str(int(APLDataFrame.loc[i,'LoadOrder']))
APLDataFrame.loc[i,'RecordCount']=str(int(APLDataFrame.loc[i,'RecordCount']))
ViewLogDisplay.config(text='--------------------------------------------------------------------------------'
'-----------------------------------------------\n'
'Index\tOrder\t'+
'FileName'.ljust(MaxLength,'_')
+'\tCount\tStartTime\t\tDuration\n'+
'--------------------------------------------------------------------------------'
'-----------------------------------------------\n'+
APLDataFrame.to_csv(sep='\t',index=False,header=False,
columns=['LoadIndex','LoadOrder','FileName','RecordCount','LoadStartDTTM','Duration'])
+'--------------------------------------------------------------------------------'
'-----------------------------------------------')
#Load Step info from Main Log
def LoadStep(ChoiceLabel):
ChoiceLabel.config(text='Load Step Logs -')
var.set('')
LoadStepDataFrame=FirebirdDB.DisplayLoadStepLog()
LoadStepDataFrame['StatusLen']=LoadStepDataFrame.Status.map(len)
MaxStatus=0
for i in range(0,len(LoadStepDataFrame)):
if LoadStepDataFrame.loc[i,'StatusLen'] > MaxStatus:
MaxStatus=LoadStepDataFrame.loc[i,'StatusLen']
for i in range(0,len(LoadStepDataFrame)):
LoadStepDataFrame.loc[i,'Status']=str(LoadStepDataFrame.loc[i,'Status']).ljust(MaxStatus,'_')
LoadStepDataFrame.loc[i,'LoadIndex']=str(int(LoadStepDataFrame.loc[i,'LoadIndex']))
LoadStepDataFrame.loc[i,'LoadOrder']=str(int(LoadStepDataFrame.loc[i,'LoadOrder']))
LoadStepDataFrame.loc[i,'RecordCount']=str(int(LoadStepDataFrame.loc[i,'RecordCount']))
LoadStepDataFrame.loc[i,'LoadStartDTTM']=datetime.datetime.strptime(LoadStepDataFrame.loc[i,'LoadStartDTTM'], "%Y-%m-%d %H:%M:%S.%f").strftime("%d %b %y %H:%M:%S")
LoadStepDataFrame.loc[i,'LoadStartDTTM']=str(LoadStepDataFrame.loc[i,'LoadStartDTTM']).ljust(30,' ').upper()
m,s=divmod(LoadStepDataFrame.loc[i,'Duration'],60)
h,m=divmod(m,60)
DurationTime=str(int(h)).rjust(2,'0')+':'+str(int(m)).rjust(2,'0')+':'+str(int(s)).rjust(2,'0')
LoadStepDataFrame.loc[i,'Duration']=DurationTime
ViewLogDisplay.config(text='--------------------------------------------------------------------------------'
'----------------------------------------------------------\n'
'Index\tOrder\t'+'Status'.ljust(MaxStatus,'_')+'\tCount\tStartTime\t\tDuration\n'+
'--------------------------------------------------------------------------------'
'----------------------------------------------------------\n'+
LoadStepDataFrame.to_csv(sep='\t',index=False,header=False,
columns=['LoadIndex','LoadOrder','Status','RecordCount','LoadStartDTTM','Duration'])
+'--------------------------------------------------------------------------------'
'----------------------------------------------------------')
#Display Log details from Archived files/Rejects
def ProcessLog():
ChoiceLabel.config(text='Archived Process Logs -')
ViewLogDisplay.config(text='Please Select an Archive File !!!')
var.set('')
#Clear the View Log Screen and revert to Default Main Page
def DisplayLogClear(root, photo, ImageFrame, MainPageFrame):
ImageFrame.destroy()
MainPageFrame.destroy()
MainPage(root, photo)
#Buttions to choose log type
APILogButton = ttk.Button(BottopViewFrame, text='API Logs', width=20,
command=lambda:APILog(ChoiceLabel))
LoadStepButton = ttk.Button(BottopViewFrame, text='Load Step Logs', width=20,
command=lambda:LoadStep(ChoiceLabel))
ProcessLogButton = ttk.Button(BottopViewFrame, text='Process Logs/Rejects', width=20,
command=ProcessLog)
BackButton = ttk.Button(BottopViewFrame, text='Back', width=20,
command=lambda: DisplayLogClear(root, photo, ImageFrame, MainPageFrame))
#pack Buttons
APILogButton.pack(side=LEFT,padx=20)
LoadStepButton.pack(side=LEFT,padx=20)
ProcessLogButton.pack(side=LEFT,padx=20)
BackButton.pack(side=LEFT,padx=20)
def FetchData(root, ImageFrame, ImageLabel, photo, MainPageFrame,FetchDataButton,ViewLogButton):
#Start the Fetching Process
InitialParameters = Initialize.getParam()
LoadFileName = InitialParameters[15]
#Disable the Button and destroy the Image
FetchDataButton.config(state=DISABLED)
ViewLogButton.config(state=DISABLED)
ImageLabel.destroy()
#Create Top and Bottom Frame
TopFetchFrame=Frame(ImageFrame)
TopFetchFrame.pack(side=TOP)
BottopFetchFrame=Frame(ImageFrame)
BottopFetchFrame.pack(side=BOTTOM)
#Label to Display Log with Scroll Bar
#Function for Scroll Bar
def ScrollLogFunc(event):
FetchCanvas.configure(scrollregion=FetchCanvas.bbox("all"),height=370,width=700)
#canvas to Implement Scroll Bar for Log
FetchCanvas=Canvas(TopFetchFrame)
CanvasFrame=Frame(FetchCanvas)
ScrollLogY=Scrollbar(TopFetchFrame, orient="vertical",command=FetchCanvas.yview)
FetchCanvas.configure(yscrollcommand=ScrollLogY.set)
FetchCanvas.pack(side=LEFT)
ScrollLogY.pack(side=RIGHT,fill=Y)
FetchCanvas.create_window((0,0),window=CanvasFrame,anchor=NW)
#Label to display the Log Details
LogContentDisplay=Label(CanvasFrame,text='Loading Log Details . . ..',anchor=NW,justify=LEFT,wraplength=650)
CanvasFrame.bind("<Configure>",ScrollLogFunc)
LogContentDisplay.pack()
#Function to Fetch Log Details from Log File - continuesly
def DisplayLogContent():
#global LogValue
CurrentLogValue=''
InitialParameters = Initialize.getParam()
LogFileName = InitialParameters[14]
try:
LogFile = open(LogFileName, 'r')
except Exception as e:
#print('Log File Error : ',e)
CurrentLogValue='Unable to Display Log.Please Wait . . .'
else:
CurrentLogValue = LogFile.read()
LogContentDisplay.config(text=CurrentLogValue)
LogContentDisplay.after(200,DisplayLogContent)
#run the Fetch Log function
DisplayLogContent()
#Clear the Fetch Data Screen and revert to Default Main Page
def DisplayLogClear(root, photo, ImageFrame, MainPageFrame):
ImageFrame.destroy()
MainPageFrame.destroy()
MainPage(root, photo)
#Button to Clear the Screen and revert back
BackButton = ttk.Button(BottopFetchFrame, text='Back', width=20,
command=lambda: DisplayLogClear(root, photo, ImageFrame, MainPageFrame))
BackButton.pack(side=LEFT,padx=50)
#Runthe Update Process using thread
if os.path.isfile(LoadFileName) == True:
popupmsg('Update Already in Progress !!!')
else:
Choice=messagebox.askquestion("Update", "Are You sure you want to update the Main DB Tables?", icon='warning')
if Choice=='yes':
threading.Thread(target=FD.main).start()
else:
DisplayLogClear(root, photo, ImageFrame, MainPageFrame)
def LoadDataStaging():
InitialParameters = Initialize.getParam()
MainLogTable = InitialParameters[18]
MainLoadStartDTTM=str(datetime.datetime.now()).split('.')[0]
APIView = InitialParameters[3].split(',')
APICategory = InitialParameters[4].split(',')
FilterValue = InitialParameters[17].split(',')
type_param = 'csv'
StagTableName = InitialParameters[11]
TableName = InitialParameters[10]
RejectFileName = InitialParameters[9]
StagingPath = InitialParameters[2]
RejectFile = StagingPath + RejectFileName
LogTableName=InitialParameters[13]
# create a Reject List
try:
LoadStartDTTM=str(datetime.datetime.now()).split('.')[0]
RFile = open(RejectFile, 'w')
#RejectComment = "Datereceived,Product,Subproduct,Issue,Subissue,Consumercomplaintnarrative,Companypublicresponse,Company,State,ZIPcode,Submittedvia,Datesenttocompany,Companyresponsetoconsumer,Timelyresponse,Consumerdisputed,ComplaintID"
RejectComment = "Company,Companypublicresponse,Companyresponsetoconsumer,ComplaintID,Consumercomplaintnarrative,Consumerdisputed,Datereceived,Datesenttocompany,Issue,Product,State,Subissue,Submittedvia,Subproduct,Timelyresponse,ZIPcode"
RFile.write(RejectComment)
RFile.close
except Exception as e:
FailureRevert(4,e)
else:
WriteToLog('\nReject File ('+RejectFileName+') Created.')
#Update the Main Log
LoadIndex=FirebirdDB.GetLoadIndex(MainLogTable)
LoadEndDTTM=str(datetime.datetime.now()).split('.')[0]
global STEP_COUNT
FirebirdDB.UpdateMainLog(MainLogTable,LoadIndex,STEP_COUNT,'Pre-Pre',RejectFileName,'Created Reject File to Capture Rejects',0,LoadStartDTTM,LoadEndDTTM)
STEP_COUNT+=1
try:
# Truncate staging db
FirebirdDB.DeleteData(StagTableName)
except Exception as e:
FailureRevert(5,e)
else :
WriteToLog('Staging Table '+StagTableName.upper()+' Truncated. \n')
try:
#Get Load Index
LoadIndex=FirebirdDB.GetLoadIndex(LogTableName)
WriteToLog('Staging Table '+StagTableName.upper()+' Loading Initiated . . . ')
WriteToLog('------------------------------------------------------------------------------------------------------------------')
# load data from files
for i in range(0, len(APICategory)):
LoadStartDTTM=str(datetime.datetime.now()).split('.')[0]
#print('InsertData(',StagTableName,',',APICategory[i].replace("'", ""))
FirebirdDB.InsertData(StagTableName, APICategory[i].replace("'", ""))
LoadEndDTTM=str(datetime.datetime.now()).split('.')[0]
RowCount=FirebirdDB.GetRecordCount(StagTableName,FilterValue[i].replace("'", ""))
WriteToLog('Loading : '+StagTableName.upper()+' <-- '+ APICategory[i].replace("'", "")+' [Count('+str(RowCount)+')]')
FirebirdDB.UpdateLog(LogTableName,(LoadIndex+1),(i+1),APIView[i].replace("'", ""),APICategory[i].replace("'", ""),RowCount,LoadStartDTTM,LoadEndDTTM)
WriteToLog('------------------------------------------------------------------------------------------------------------------')
except Exception as e:
FailureRevert(6,e)
else:
#Update the Main Log
LoadIndex=FirebirdDB.GetLoadIndex(MainLogTable)
TableCount=FirebirdDB.GetTableCount(StagTableName)
MainLoadEndDTTM=str(datetime.datetime.now()).split('.')[0]
FirebirdDB.UpdateMainLog(MainLogTable,LoadIndex,STEP_COUNT,'Data-Load',StagTableName,'Staging Table Loaded',TableCount,MainLoadStartDTTM,MainLoadEndDTTM)
STEP_COUNT+=1
def PostLoadTasks(*RevertList):
if len(RevertList) == 0 :
RevertList=['S','T','L','R']
InitialParameters = Initialize.getParam()
MainLogTable = InitialParameters[18]
TableName = InitialParameters[10]
ViewName=InitialParameters[16]
LogPath=InitialParameters[19]
LoadFileName = InitialParameters[15]
LogFileName = InitialParameters[14]
LoadIndex=FirebirdDB.GetLoadIndex(MainLogTable)
LogTime=time.strftime("%d%b%Y_%X")
global STEP_COUNT,FAIL_FLAG
if 'S' in RevertList:
try:
LoadStartDTTM=str(datetime.datetime.now()).split('.')[0]
#Swap View
FirebirdDB.ViewSwap(TableName,ViewName)
ViewCount=FirebirdDB.GetTableCount(ViewName)
LoadEndDTTM=str(datetime.datetime.now()).split('.')[0]
FirebirdDB.UpdateMainLog(MainLogTable,LoadIndex,STEP_COUNT,'Post-Post',ViewName,'View Swap back to Main Table',ViewCount,LoadStartDTTM,LoadEndDTTM)
STEP_COUNT+=1
except Exception as e:
FailureRevert(8,e)
else :
WriteToLog('\nView Swap ('+ViewName.upper()+' --> '+TableName.upper()+') Completed.')
if 'R' in RevertList:
try:
#Checkif the Rejects Exists if so then move then to log dir
LoadStartDTTM=str(datetime.datetime.now()).split('.')[0]
RejectFileName = InitialParameters[9]
StagingPath = InitialParameters[2]
RejectFile = StagingPath + RejectFileName
if sum(1 for line in open(RejectFile))==1:
WriteToLog('Zero Reject Records Found !!! ')
RejectStatus='No Rejects Found'
TotalRejectCount=0
os.remove(RejectFile)
else:
TotalRejectCount=sum(1 for line in open(RejectFile))-1
WriteToLog('Rejects Records Captured in - '+RejectFileName+' Count: '+str(TotalRejectCount))
ArchiveReject=LogPath+'RejectFile_'+LogTime.replace(':','')+'.csv'
RejectStatus='Rejects Found...Archiving the Reject File as '+ ArchiveReject.split('\\')[-1]
shutil.move(RejectFile,ArchiveReject)
WriteToLog('Reject File ('+RejectFileName+') Sucessfully Archived.')
except Exception as e:
FailureRevert(9,e)
else:
LoadEndDTTM=str(datetime.datetime.now()).split('.')[0]
FirebirdDB.UpdateMainLog(MainLogTable,LoadIndex,STEP_COUNT,'Post-Post',RejectFileName,RejectStatus,TotalRejectCount,LoadStartDTTM,LoadEndDTTM)
STEP_COUNT+=1
if 'T' in RevertList:
try:
#Delete Trig File
LoadStartDTTM=str(datetime.datetime.now()).split('.')[0]
#remove the file
os.remove(LoadFileName)
LoadEndDTTM=str(datetime.datetime.now()).split('.')[0]
if 'F' in RevertList:
TrigStatus='Trig File deleted .Process Terminated due to Error !!!'
FirebirdDB.UpdateMainLog(MainLogTable,LoadIndex,STEP_COUNT,'Post-Post',LoadFileName.split('\\')[-1],TrigStatus,0,LoadStartDTTM,LoadEndDTTM)
STEP_COUNT+=1
WriteToLog('\nTrig File Deleted -- Process Terminated.')
else:
TrigStatus='Trig File deleted to indicate Data Load Status as Completed'
FirebirdDB.UpdateMainLog(MainLogTable,LoadIndex,STEP_COUNT,'Post-Post',LoadFileName.split('\\')[-1],TrigStatus,0,LoadStartDTTM,LoadEndDTTM)
STEP_COUNT+=1
WriteToLog('\nTrig File Deleted to Indicate Data Load Complete.')
except Exception as e:
FailureRevert(10,e)
if 'L' in RevertList:
try:
##Move log file
LoadStartDTTM=str(datetime.datetime.now()).split('.')[0]
ArchiveLog=LogPath+'DataUpdateLog_'+LogTime.replace(':','')+'.log'
WriteToLog('Log File ('+LogFileName.split('\\')[-1]+') Archived.')
if 'F' in RevertList:
WriteToLog('\nUpdate Process Terminated due to Error !!! '+time.strftime('%d %b %Y %X'))
else:
WriteToLog('\nUpdate Process Sucessfully Completed. . . '+time.strftime('%d %b %Y %X'))
WriteToLog('\n\n\n*Note-Display will be cleared in 10 seconds.')
time.sleep(10)
shutil.move(LogFileName,ArchiveLog)
LoadEndDTTM=str(datetime.datetime.now()).split('.')[0]
FirebirdDB.UpdateMainLog(MainLogTable,LoadIndex,STEP_COUNT,'Post-Post',LogFileName.split('\\')[-1],'Log File Archived as '+ArchiveLog.split('\\')[-1],0,LoadStartDTTM,LoadEndDTTM)
STEP_COUNT+=1
WriteToLog('-- Log File Cleared/Archived !!! ---')
except Exception as e:
FailureRevert(11,e)
FAIL_FLAG=True
def kmeans(k, training_data, initial_clusters, distfn = Distance.sumsq,
method = "means"):
n = len(training_data) # number of training instances
i = 0 # keep track of iteration
r = np.zeros((n,k)) # create empty array to store cluster assignments
# find and store k that minimize sum of square distance for each image
# in form of a vector of shape (n,)
newks = np.apply_along_axis(Distance.leastsquares, 1, training_data,
initial_clusters, distfn)
# Check if every cluster is represented within the cluster assignments;
# if not then re-initialize centers
while np.array_equal(np.unique(newks), np.array(range(k))) == False:
initial_clusters = Initialize.random_centers(k)
newks = np.apply_along_axis(Distance.leastsquares, 1, training_data,
initial_clusters, distfn)
# create one hot coded vector for each image to signify cluster assignment
r[range(n), newks] = 1
# create a "means" vector to store cluster centers as they are updated
means = initial_clusters
# Find new means
while True:
for smallk in range(k): # iterate through clusters
ones = np.where(r[:,smallk]==1)[0]
# The k-means method updates cluster centers to be the mean of each
# corresponding pixel of the datapoints that are contained in that
# cluster.
if method == "means":
means[smallk,:] = np.mean(training_data[list(ones),:], axis=0)
# The k-medoids method updates cluster centers to be the closest
# *datapoint* to the mean of the corresponding pixel of datapoints
# contained in that cluster.
elif method == "medoids":
dist_to_ctr = np.sum((training_data[list(ones),:]
- np.mean(training_data[list(ones),:], axis=0))**2,axis=1)
means[smallk,:]=training_data[list(ones),:]\
[np.argmin(dist_to_ctr)]
# The k-medians method updates cluster centeras to be the median of
# each corresponding pixel of the datapoints contained in that
# cluster.
elif method == "medians":
means[smallk,:] = np.median(training_data[list(ones),:],axis=0)
# If no proper value is chosen for method, then return error.
else:
raise ValueError("Not a valid method specification; must be \
'means','medoids', or 'medians'")
# update responsibilities by minimizing distance metric
r_new = np.zeros((n,k))
# stores indices of k's that minimize distance metric
newks = np.apply_along_axis(Distance.leastsquares, 1, training_data,
means, distfn)
r_new[range(n), newks] = 1
# if none of the responsibilities change, then we've reached the optimal
# cluster assignments
if np.all((r_new - r)==0):
return r, means
else:
r = r_new
# After each iteration, print iteration number and the number of images
# assigned to a given cluster.
print i, r.sum(axis=0)
i += 1
print 'finished'
