def __init__(self, caption="Trans:Coda Editor"):
super().__init__()
self.encoder = EncoderSelector()
self.encoder_view = EncoderView()
self.actions = QToolBar()
self.caption = caption
self.add_action = CommonUtils.create_action(
tooltip="Add Encoder",
icon=TransCoda.theme.ico_add_item,
func=self.action_add,
name="Add",
parent=self)
self.del_action = CommonUtils.create_action(
tooltip="Delete Encoder",
icon=TransCoda.theme.ico_clear,
func=self.action_del,
name="Delete",
parent=self)
self.mod_action = CommonUtils.create_action(
tooltip="Edit Encoder",
icon=TransCoda.theme.ico_edit,
func=self.action_mod,
name="Edit",
parent=self)
self.ref_action = CommonUtils.create_action(
tooltip="Refresh Encoder",
icon=TransCoda.theme.ico_refresh,
func=self.action_ref,
name="Refresh",
parent=self)
self._init_ui()
def __init__(self, task_executor):
super().__init__()
self.lib_selector = QComboBox()
self.libraries = self._setup_libraries(self.lib_selector)
self.lib_selector.currentIndexChanged.connect(self._lib_changed)
self.info_ico_lbl = QLabel("ico")
self.info_lbl = QLabel("Library Information")
self.info_lbl.setStyleSheet('font-size: 8pt;')
self.task_executor = task_executor
self.group_manager = LibraryGroupingManager()
self.treeView = QTreeView()
self.toolbar = QToolBar()
self.refresh_action = CommonUtils.create_toolbar_action(
"Refresh/Rescan this Library", QIcon.fromTheme("view-refresh"),
self.refresh)
self.edit_action = CommonUtils.create_toolbar_action(
"Configure this Library", QIcon.fromTheme("preferences-system"),
self.edit)
self.delete_action = CommonUtils.create_toolbar_action(
"Delete this Library", QIcon.fromTheme("list-remove"), self.delete)
self.add_action = CommonUtils.create_toolbar_action(
"Add a new Library", QIcon.fromTheme("list-add"), self.add)
self._init_ui()
def elevatorHeartBeat(self, data, client):
'''
梯控心跳
:return:
'''
requestBody = {
'sourceIp': client[0],
'sourcePort': client[1],
'mac': CommonUtils.bytesToHex(data[0:6]),
'targetIp': '' if not Config.udpServerIp else Config.udpServerIp,
'targetPort': Config.udpServerPort,
'date': CommonUtils.now(),
}
logger.debug("梯控心跳:%s" % (requestBody))
response = HttpClient.doPost(Config.elevatorHeartbeat, requestBody)
logger.debug('梯控心跳结果:%s' % (response))
if response['success']:
cardFormat = response['data']['cardFormat']
key = response['data']['key']
if not key:
logger.error('没有密钥')
return
if not cardFormat:
cardFormat = '04'
message = requestBody['mac'] + '17' + key + time.strftime(
"%y%m%d%H%M%S") + cardFormat
message = CommonUtils.xorOperation(bytes.fromhex(message))
logger.debug('梯控心跳响应:%s' % (CommonUtils.bytesToHex(message)))
UdpClient.sendUdpData(message, client)
def elevatorTempPassValidate(self, data, client):
'''
梯控临时密码校验
:param data:
:param celint:
:return:
'''
requestBody = {
'serial': CommonUtils.bytesToHex(data[0:6]),
'pass': CommonUtils.bytesToHex(data[7:9]),
}
logger.debug('梯控临时密码校验:%s' % (requestBody))
response = HttpClient.doPost(Config.elevatorTempPassValidate,
requestBody)
logger.debug('梯控临时密码校验结果:%s' % (response))
if response['success']:
floors = response['data']
message = None
if floors:
# 存在一个或者多个楼层权限
message = requestBody[
'serial'] + '15' + CommonUtils.getFloorHex(floors) + '4F4B'
else:
# 校验失败,不存在楼层权限
message = requestBody['serial'] + '15' + '4445'
message = CommonUtils.xorOperation(bytes.fromhex(message))
logger.debug('梯控临时密码响应:%s' % (CommonUtils.bytesToHex(message)))
UdpClient.sendUdpData(message, client)
def uploadPassRecord(self, data, client):
'''
密码开门记录上传
:param data:
:param client:
:return:
'''
requestBody = {
'serial': CommonUtils.bytesToHex(data[0:6]),
'pass': CommonUtils.bytesToHex(data[7:9]),
}
logger.debug('上传密码开门记录:%s' % (requestBody))
response = HttpClient.doGet(Config.doorRecord + '?type=2&mac=' +
requestBody['serial'] + '&cardNum=' +
requestBody['pass'])
logger.debug('密码开门记录上传结果:%s' % (response))
data = requestBody['serial'] + '0A4F4B'
data = CommonUtils.xorOperation(bytes.fromhex(data))
logger.debug('响应UDP门禁:hex=%s' % (CommonUtils.bytesToHex(data)))
UdpClient.sendUdpData(data, client)
def setCards(self,requestBody):
'''
电梯下发/删除卡号
:param requestBody:
:return:
'''
serial = requestBody['serial']
ip = requestBody['ip']
port = int(requestBody['port'])
cards = requestBody['cards']
if cards:
for card in cards:
num = card['num'].zfill(10)
operation = card['operation']
floors = card.get('floors',[])
message = serial
if operation == "create":
# 下发卡
message += '12' + num + CommonUtils.getFloorHex(floors)
else:
# 删除卡
message += '13' + num
message = CommonUtils.xorOperation(bytes.fromhex(message))
logger.debug('梯控卡号下发/删除:%s'%(CommonUtils.bytesToHex(message)))
UdpClient.sendUdpData(message,(ip,port))
def uploadRecord(self, data, client):
'''
刷卡记录上传
:param data:
:param client:
:return:
'''
requestBody = {
'serial': CommonUtils.bytesToHex(data[0:6]),
'carNum': CommonUtils.bytesToHex(data[7:12]),
'dateTime': CommonUtils.bytesToHex(data[12:])
}
logger.debug('上传刷卡记录:%s' % (requestBody))
response = HttpClient.doGet(Config.doorRecord + '?type=1&mac=' +
requestBody['serial'] + '&cardNum=' +
requestBody['carNum'])
logger.debug('刷卡记录上传结果:%s' % (response))
data = requestBody['serial'] + '01' + requestBody['carNum']
if response['success']:
data += '4F4B'
else:
data += '4445'
data = CommonUtils.xorOperation(bytes.fromhex(data))
logger.debug('响应UDP门禁:hex=%s' % (CommonUtils.bytesToHex(data)))
UdpClient.sendUdpData(data, client)
def parseJobInfo(container, filename=None):
if filename:
cmd = shlex.split('cat ' + filename)
else:
cmd = shlex.split('scontrol -o show jobs')
container = JobInfoHandler(container)
CommonUtils.parseStream(cmd, container)
def parseCPUInfo(filename=None):
if filename:
cmd = shlex.split('cat ' + filename)
else:
cmd = shlex.split('scontrol -o show nodes')
container = NodesInfoHandler()
CommonUtils.parseStream(cmd, container)
return container.ncpu, container.nfree
def parseConfiguration(filename=None):
if filename:
cmd = shlex.split('cat ' + filename)
else:
cmd = shlex.split('scontrol show config')
container = ConfigInfoHandler()
CommonUtils.parseStream(cmd, container)
return container
def __init__( self , session ):
"""Input: coral session handle
"""
self.__session = session
self.__tagInventoryTableName=CommonUtils.inventoryTableName()
self.__tagInventoryIDName=CommonUtils.inventoryIDTableName()
self.__tagInventoryTableColumns = {'tagid':'unsigned long', 'tagname':'string', 'pfn':'string','recordname':'string', 'objectname':'string', 'labelname':'string'}
self.__tagInventoryTableNotNullColumns = ['tagname','pfn','recordname','objectname']
#self.__tagInventoryTableUniqueColumns = ['tagname']
self.__tagInventoryTablePK = ('tagid')
def parsePartInfo(filename=None):
if filename:
cmd = shlex.split('cat ' + filename)
else:
cmd = shlex.split('sinfo -h -o "%20P %5a %25C %25l %25L %25s %25F %25B %25z"')
container = PartitionInfoHandler()
CommonUtils.parseStream(cmd, container)
return container
def parsePartInfo(filename=None):
if filename:
cmd = shlex.split('cat ' + filename)
else:
cmd = shlex.split('scontrol -o show partitions')
container = PartitionInfoHandler()
CommonUtils.parseStream(cmd, container)
return container
def parsePartInfo(filename=None):
if filename:
cmd = shlex.split('cat ' + filename)
else:
cmd = shlex.split(
'sinfo -h -o "%20P %5a %25C %25l %25L %25s %25F %25B %25z"')
container = PartitionInfoHandler()
CommonUtils.parseStream(cmd, container)
return container
def corr():
'''
plot correlation matrix for the fit parameters
'''
corr=[np.corrcoef(np.load("../DATA/misc/pcov_S12.np.npy")), np.corrcoef(np.load("../DATA/misc/pcov_S18.np.npy"))]
for i_station, station in enumerate(stations):
df_corr = pd.DataFrame(corr[i_station],columns=par_labels, index=par_labels)
fig,ax = plt.subplots()
ax=sn.heatmap(df_corr, annot=True, fmt='.2f', linewidths=.5, cmap="bwr")
cu.textL(ax, 0.5, 1.1, "S"+str(station))
fig.savefig("../fig/corr_S"+str(station)+".png", dpi=300)
def main():
test_blidning(n_inject)
ax, legend = cu.plotHist(idiff_array)
plt.xlim(-1, 10)
ax.set_xlabel(xlabel=r"$R_{blind}-R_{ref}$ (ppm)")
ax.set_ylabel(ylabel="")
cu.textL(ax, 0.8, 0.85, str(legend), font_size=14)
plt.savefig("../fig/edm_test.png", dpi=300)
np.set_printoptions(precision=15)
np.save("../fig/pyEDMBlind.npy", idiff_array)
def setElevatorCardFormat(self,requestBody):
'''
下发卡号输出格式 在心跳里面包含有,该接口不做实现
:param requestBody:
:return:
'''
serial = requestBody['serial']
ip = requestBody['ip']
port = int(requestBody['port'])
cardFormat = requestBody['cardFormat']
message = serial + '18' + cardFormat
message = CommonUtils.xorOperation(bytes.fromhex(message))
logger.debug('设置梯控卡号输出格式:%s' % (CommonUtils.bytesToHex(message)))
UdpClient.sendUdpData(message, (ip, port))
def setElevatorKey(self,requestBody):
'''
下发密钥 密钥在心跳数据里面已经包含,该接口不做实现
:param requestBody:
:return:
'''
serial = requestBody['serial']
ip = requestBody['ip']
port = int(requestBody['port'])
key = requestBody['key']
message = serial + '16' + key
message = CommonUtils.xorOperation(bytes.fromhex(message))
logger.debug('设置梯控密钥:%s'%(CommonUtils.bytesToHex(message)))
UdpClient.sendUdpData(message,(ip,port))
def isThereDifference(businessDate, OneDayTransaction):
cashTans = OneDayTransaction.CashAtTheEnd
nonCashTrans = OneDayTransaction.CardPaytm + OneDayTransaction.Swiggy
expense = OneDayTransaction.CashExpenses
difference = OneDayTransaction.TotalSale - (cashTans +
nonCashTrans) - expense
if difference == 0:
CommonUtils.logging(
"Entered Transaction data matches for {}".format(businessDate))
else:
CommonUtils.logging("Amount Differing for {} is Rs.{}.00".format(
businessDate, int(difference)))
def heartbeatWithStatus(self, data, client):
'''
带有状态的心跳包
:param data:
:param client:
:return:
'''
requestBody = {
'sourceIp': client[0],
'sourcePort': client[1],
'mac': CommonUtils.bytesToHex(data[0:6]),
'targetIp': '' if not Config.udpServerIp else Config.udpServerIp,
'targetPort': Config.udpServerPort,
'status': CommonUtils.bytesToHex(data[7:8]),
'date': CommonUtils.now(),
}
logger.debug('状态心跳:%s' % (requestBody))
response = HttpClient.doPost(Config.doorHeartbeat, requestBody)
logger.debug('状态响应:%s' % (response))
if not response['success']:
logger.error('服务器异常:%s' % (response['message']))
return
if 'data' not in response or not response['data']:
logger.error('门禁没有密钥和卡片输出格式:serital=%s' % (requestBody['mac']))
return
if 'key' not in response['data']:
logger.error('门禁没有密钥信息:serital=%s' % (requestBody['mac']))
return
key = response['data']['key']
if 'cardOutFormat' not in response['data']:
cardFormat = '04'
else:
cardFormat = response['data']['cardOutFormat'].zfill(2)
responseHex = requestBody['mac'] + '07' + key + time.strftime(
"%y%m%d%H%M%S") + cardFormat
data = CommonUtils.xorOperation(bytes.fromhex(responseHex))
logger.debug('状态响应包:hex=%s' % (CommonUtils.bytesToHex(data)))
UdpClient.sendUdpData(data, client)
def timeAsync(self,requestBody):
'''
主动时间同步
:param requestBody:
:return:
'''
serial = requestBody['serial']
ip = requestBody['ip']
port = int(requestBody['port'])
message = serial + '14' + time.strftime("%y%m%d%H%M%S")
message = CommonUtils.xorOperation(bytes.fromhex(message))
logger.debug('梯控时间同步:%s'%(CommonUtils.bytesToHex(message)))
UdpClient.sendUdpData(message, (ip, port))
def setElevatorPass(self,requestBody):
'''
设置电梯固定密码
:param requestBody:
:return:
'''
serial = requestBody['serial']
ip = requestBody['ip']
port = int(requestBody['port'])
_pass = requestBody['pass']
message = serial + '1B' + _pass
message = CommonUtils.xorOperation(bytes.fromhex(message))
logger.debug('设置电梯固定密码:%s'%(CommonUtils.bytesToHex(message)))
UdpClient.sendUdpData(message,(ip,port))
def elevatorControll(self,requestBody):
'''
电梯控制
:return:
'''
serial = requestBody['serial']
ip = requestBody['ip']
port = int(requestBody['port'])
floors = requestBody['floors']
message = serial + '10' + CommonUtils.getFloorHex(floors)
message = CommonUtils.xorOperation(bytes.fromhex(message))
logger.debug('执行电梯控制:%s'%(CommonUtils.bytesToHex(message)))
UdpClient.sendUdpData(message,(ip,port))
def insertPlayer(players):
insertSql = "insert into player(player_zn_name,player_eng_name,team,location,height,weight,birthday,money) " \
"values(""?,?,?,?,?,?,?,?)"
selectSql = "select * from player where player_eng_name = ? and birthday=?"
updateSql = "update player set team=?,location=?,height =?,weight=?,birthday=?,money=?,player_zn_name=? where player_eng_name=? and birthday=?"
conn = CommonUtils.getConnect()
cu = conn.cursor()
#cu.execute("delete from player")
insertValues = []
updateValues = []
for player in players:
# 先判断是否存在此球员
cu.execute(selectSql, (player.playerEngName, player.birthday))
result = cu.fetchall()
if(len(result) == 0):
#若不存在，则直接插入
insertValues.append((player.playerZnName, player.playerEngName, player.teamName, player.location, player.height, player.weight, player.birthday, player.money))
else:
# 若存在，则直接更新即可
updateValues.append((player.teamName,player.location,player.height,player.weight,player.birthday,player.money,player.playerZnName,player.playerEngName,player.birthday))
# before insert,delete data that is older
#cu.execute("delete from player")
# insert player data
print "新增数据" + str(len(insertValues))
print "更新数据" + str(len(updateValues))
cu.executemany(insertSql,insertValues)
cu.executemany(updateSql,updateValues)
conn.commit()
conn.close()
def card(self,requestBody):
'''
卡片同步执行
:param requestBody:
:return:
'''
logger.debug('执行卡片同步:%s'%(requestBody))
ip = requestBody['ip']
port = int(requestBody['port'])
serial = requestBody['serial']
cards = requestBody['cards']
if cards:
for card in cards:
cardNum = card['num']
operation = card['operation']
message = serial
if operation == 'create':
message += '02'
elif operation == 'delete':
message += '03'
else:
continue
cardNum = cardNum.zfill(10)
message += cardNum
logger.debug('同步卡片信息(%s):%s'%('添加' if operation == 'create' else '删除',message))
message = CommonUtils.xorOperation(bytes.fromhex(message))
# 发送udp包到设备
UdpClient.sendUdpData(message,(ip,port))
return responseBody['success']
def getCommentsForTable(self, tableName):
"""get all comments for given table
result=[(entryid,comment)]
"""
transaction = self.__session.transaction()
result = []
try:
transaction.start(True)
schema = self.__session.nominalSchema()
query = schema.tableHandle(
CommonUtils.commentTableName()).newQuery()
condition = 'tablename = :tablename'
conditionbindDict = coral.AttributeList()
conditionbindDict.extend('tablename', 'string')
conditionbindDict['tablename'].setData(tableName)
query.addToOutputList('entryid')
query.addToOutputList('comment')
query.setCondition(condition, conditionbindDict)
cursor = query.execute()
while next(cursor):
comment = cursor.currentRow()['comment'].data()
entryid = cursor.currentRow()['entryid'].data()
result.append((entryid, comment))
cursor.close()
transaction.commit()
del query
return result
except Exception, e:
transaction.rollback()
raise Exception, str(e)
def getCommentForId(self, tableName, entryid):
"""get comment for given id in given table
"""
transaction = self.__session.transaction()
comment = ''
try:
transaction.start(True)
schema = self.__session.nominalSchema()
query = schema.tableHandle(
CommonUtils.commentTableName()).newQuery()
condition = 'entryid = :entryid AND tablename = :tablename'
conditionbindDict = coral.AttributeList()
conditionbindDict.extend('entryid', 'unsigned long')
conditionbindDict.extend('tablename', 'string')
conditionbindDict['entryid'].setData(entryid)
conditionbindDict['tablename'].setData(tableName)
query.addToOutputList('comment')
query.setCondition(condition, conditionbindDict)
cursor = query.execute()
if next(cursor):
comment = cursor.currentRow()['comment'].data()
cursor.close()
transaction.commit()
del query
return comment
except Exception, e:
transaction.rollback()
raise Exception, str(e)
def downloadThumbnail(self):
"""
下载缩略图
"""
cacheDir = "cache" + os.path.sep + self._infoType + os.path.sep + "img" + os.path.sep + str(self._newsId)
url = self._thumbnail
thumbnailPath = self.requestUrlContent(url, cacheDir, CommonUtils.md5(url) + os.path.splitext(url)[-1])
if not os.path.isfile(thumbnailPath):
return None
# 处理成5种尺寸, 按宽度自适应
# XS:100, S:200, M:400, L:600, XL:900
im = Image.open(thumbnailPath)
width, height = im.size
filename, ext = os.path.splitext(os.path.basename(thumbnailPath))
thumbnailCachePath = self._thumbnailCachePath
# XS
target_file = os.path.join(thumbnailCachePath, filename+"_portfolio_"+"XS"+ext)
try:
_width = 100
_heigh = _width * height / width
out = im.resize((_width, _heigh), Image.ANTIALIAS)
out.save(target_file)
except Exception, e:
shutil.copyfile(thumbnailPath, target_file)
def getCommentsForTable( self, tableName ):
"""get all comments for given table
result=[(entryid,comment)]
"""
transaction=self.__session.transaction()
result=[]
try:
transaction.start(True)
schema = self.__session.nominalSchema()
query = schema.tableHandle(CommonUtils.commentTableName()).newQuery()
condition='tablename = :tablename'
conditionbindDict=coral.AttributeList()
conditionbindDict.extend('tablename','string')
conditionbindDict['tablename'].setData(tableName)
query.addToOutputList('entryid')
query.addToOutputList('comment')
query.setCondition(condition,conditionbindDict)
cursor=query.execute()
while cursor.next():
comment=cursor.currentRow()['comment'].data()
entryid=cursor.currentRow()['entryid'].data()
result.append((entryid,comment))
cursor.close()
transaction.commit()
del query
return result
except Exception, e:
transaction.rollback()
raise Exception, str(e)
def delay(self,requestBody):
'''
设置开门延迟
:param requestBody:
:return:
'''
logger.debug('设置开门延迟:%s'%(requestBody))
ip = requestBody['ip']
port = int(requestBody['port'])
serial = requestBody['serial']
delay = requestBody['delay'].zfill(2)
message = serial + '09' + delay
message = CommonUtils.xorOperation(bytes.fromhex(message))
logger.debug('延迟设置指令:%s'%(CommonUtils.bytesToHex(message)))
UdpClient.sendUdpData(message,(ip,port))
return responseBody['success']
def requestUrlContent(url, cache_dir=None, filename=None):
if cache_dir != None and not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
ext = os.path.splitext(url)[-1]
if filename == None:
filename = CommonUtils.md5(url) + ext
target_path = None
if cache_dir == None:
target_path = tempfile.mktemp()
else:
target_path = os.path.join(".", cache_dir)
target_path = os.path.join(target_path, filename)
if target_path == None:
target_path = tempfile.mktemp()
wget = 'wget --user-agent="Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)"'
command = wget + ' "%s" -O %s --timeout=60 --tries=2'%(url, target_path)
if (not os.path.isfile(target_path)):
state = os.system(command.encode("utf-8")) # 在windows下是gb2312, 在ubuntu主机上应该是utf-8
if state != 0:
print url, "download failed!"
else:
pass
# 需要保证返回的字符串是ascii编码，否则lxml xpath解析可能会出问题
# phantomjs保存的文件是utf-8编码，所以需要进行转码为ascii编码
return open(target_path).read()
def getCommentForId( self, tableName, entryid ):
"""get comment for given id in given table
"""
transaction=self.__session.transaction()
comment=''
try:
transaction.start(True)
schema = self.__session.nominalSchema()
query = schema.tableHandle(CommonUtils.commentTableName()).newQuery()
condition='entryid = :entryid AND tablename = :tablename'
conditionbindDict=coral.AttributeList()
conditionbindDict.extend('entryid','unsigned long')
conditionbindDict.extend('tablename','string')
conditionbindDict['entryid'].setData(entryid)
conditionbindDict['tablename'].setData(tableName)
query.addToOutputList('comment')
query.setCondition(condition,conditionbindDict)
cursor=query.execute()
if cursor.next():
comment=cursor.currentRow()['comment'].data()
cursor.close()
transaction.commit()
del query
return comment
except Exception, e:
transaction.rollback()
raise Exception, str(e)
def Run_Pipeline(args):
"""
The data (ellipsoid.zip) is dowloaded to /Data folder and extracted to
/Output/ellpsoid_cut
"""
print("\nStep 1. Extract Data\n")
if int(args.interactive) != 0:
input("Press Enter to continue")
# Get data
datasetName = "ellipsoid-v0"
outputDirectory = "Output/ellipsoid_cut/"
if not os.path.exists(outputDirectory):
os.makedirs(outputDirectory)
CommonUtils.download_and_unzip_dataset(datasetName, outputDirectory)
fileList = sorted(glob.glob(outputDirectory + datasetName + "/segmentations/*.nrrd"))
# Select data for tiny test
if args.tiny_test:
args.use_single_scale = 1
fileList = fileList[:3]
# Select data if using subsample
if args.use_subsample:
sample_idx = sampledata(fileList, int(args.num_subsample))
fileList = [fileList[i] for i in sample_idx]
else:
sample_idx = []
"""
## GROOM : Data Pre-processing
For the unprepped data the first few steps are
-- Isotropic resampling
-- Center
-- Padding
-- Center of Mass Alignment
-- Rigid Alignment
-- Largest Bounding Box and Cropping
For a detailed explanation of grooming steps see: /Documentation/Workflow/Groom.md
"""
if args.skip_grooming:
print("Skipping grooming.")
dtDirecory = outputDirectory + datasetName + '/groomed/distance_transforms/'
indices = []
if args.tiny_test:
indices = [0,1,2]
elif args.use_subsample:
indices = sample_idx
dtFiles = CommonUtils.get_file_list(dtDirecory, ending=".nrrd", indices=indices) else:
def Run_Pipeline(args):
if args.tiny_test:
print("\nRunning a tiny test.")
print("\nStep 1. Get Data") #############################################################################################
'''
Get femur data
Femur data is downloaded in a zip folder to Data/
It gets extracted to Output/deep_ssm
'''
datasetName = "femur-v0"
outputDirectory = "Output/deep_ssm/"
if not os.path.exists(outputDirectory):
os.makedirs(outputDirectory)
CommonUtils.download_and_unzip_dataset(datasetName, outputDirectory)
input_dir = outputDirectory + datasetName + '/'
# Get image path list
img_dir = input_dir + "groomed/images/"
img_list = []
for file in os.listdir(img_dir):
img_list.append(img_dir + file)
img_list = sorted(img_list)
# Get particles path list
model_dir = input_dir + "shape_models/femur/1024/"
world_particle_list = []
local_particle_list = []
for file in os.listdir(model_dir):
if "local" in file:
local_particle_list.append(model_dir + file)
if "world" in file:
world_particle_list.append(model_dir + file)
local_particle_list = sorted(local_particle_list)
world_particle_list = sorted(world_particle_list)
# split into train and test
train_img_list = img_list[:10]
train_local_particle_list = local_particle_list[:10]
train_world_particle_list = world_particle_list[:10]
num_samples = 10
num_dim = 6
percent_variability=0
sampler_type = "kde"
embedded_dim = DataAugmentationUtils.runDataAugmentation(outputDirectory + "Augmentation/", train_img_list, train_local_particle_list, num_samples, num_dim, percent_variability, sampler_type, mixture_num=0, processes=4, world_point_list=train_world_particle_list)
aug_data_csv = outputDirectory + "Augmentation/TotalData.csv"
viz_type = "violin"
DataAugmentationUtils.visualizeAugmentation(aug_data_csv, viz_type)
def setElevatorConfig(self,requestBody):
'''
设置电梯参数格式
:param requestBody:
:return:
'''
serial = requestBody['serial']
ip = requestBody['ip']
port = int(requestBody['port'])
delay = requestBody['delay'] .zfill(2) # 开门延迟
switch = requestBody['switch'].zfill(2) # 行程开关号
message = serial + '19' + delay + switch
message = CommonUtils.xorOperation(bytes.fromhex(message))
logger.debug('设置电梯参数格式:%s'%(CommonUtils.bytesToHex(message)))
UdpClient.sendUdpData(message, (ip, port))
def doorSetting(self,requestBody):
'''
执行udp门禁远程设置
:param requestBody:
:return:
'''
logger.debug('执行远程设置:%s' % (requestBody))
ip = requestBody['ip'] # 目标IP
port = int(requestBody['port']) # 目标端口
operation = int(requestBody['type']) # 操作类型
header = requestBody['header'].zfill(2) # 机号
param = requestBody['param'].zfill(2) # 设置参数
do = requestBody['do'].zfill(2) # 继电器动作(operation)
message = None
if operation == 1: # 设备初始化 03{0}0A
template = '03{0}0A'
message = template.format(header)
elif operation == 2: # 设置机号 05FF71{0}{1}
template = '05FF71{0}{1}'
message = template.format(param, CommonUtils.getHexComplement(param))
elif operation == 3: # 设置开门延迟 05{0}0E{1}{2}
template = '05{0}0E{1}{2}'
message = template.format(header, param, do)
elif operation == 4: # 设置扇区读写指令 09{0}11{1}
template = '09{0}11{1}'
message = template.format(header, param)
elif operation == 5: # 设置扇区区号 05{0}05{1}{2}
template = ' 05{0}05{1}{2}'
message = template.format(header, param, CommonUtils.getHexComplement(param))
if not message:
return
message = message.upper()
logger.debug("指令渲染结果:%s" % (message))
message = CommonUtils.xorOperation(bytes.fromhex(message))
logger.debug("指令运算结果:%s" % (CommonUtils.bytesToHex(message)))
UdpClient.sendUdpData(message,(ip, port))
return responseBody['success']
def howMuchCashNotDepositedToBank(Last30Days):
cashNotDepositedToBank = 0
prevBD = CommonUtils.getDate(-1)
for dayBefore in range(1, 30):
dayBefore *= -1
forBusinessDay = CommonUtils.getDate(dayBefore)
eachDayTran = Transaction(forBusinessDay, Last30Days[forBusinessDay])
cashNotDepositedToBank += eachDayTran.CashAtTheEnd
if eachDayTran.bankDepositDoneOnThisDay():
break
if cashNotDepositedToBank > 0:
CommonUtils.logging \
("Cash to be deposited to bank as of {} is Rs.{}.00".format(prevBD, int(cashNotDepositedToBank)))
else:
CommonUtils.logging(
"Cash deposit to bank as of {} is upto date".format(prevBD))
def heartbeat(self, data, client):
'''
普通心跳
:param data:
:param client:
:return:
'''
requestBody = {
'sourceIp': client[0], # 门禁ip
'sourcePort': client[1], # 门禁端口
'mac': CommonUtils.bytesToHex(data), # 序列号
'targetIp': Config.udpServerIp, # 目标服务器ip(当前服务器ip)
'targetPort': Config.udpServerPort, # 目标服务器端口
'date': CommonUtils.now(), # 心跳时间
}
logger.debug('心跳注册:%s' % (requestBody))
HttpClient.doPost(Config.doorHeartbeat, requestBody)
def open_paths_in_system_default(self, item_indices, attr, func=None):
locations = set()
for index in item_indices:
item = self.main_panel.get_items(index)
if hasattr(item, attr):
if func:
locations.add(func(getattr(item, attr)))
else:
locations.add(getattr(item, attr))
if len(locations) > 3:
choice = QMessageBox.question(self, "Are you Sure?",
f"You are attempting to open several directories. "
f"Are you sure you want to proceed?")
if choice == QMessageBox.No:
return
for loc in locations:
CommonUtils.open_file_item(loc)
def retrieveWeights(modelBuilder, weightsPath, tpu_address):
tfVersion = utils.tfVersion()
if tfVersion["MAJOR"] > 1:
return __retrieveWeights200(modelBuilder, weightsPath, tpu_address)
else:
if tfVersion["MINOR"] > 13:
return __retrieveWeights114(modelBuilder, weightsPath, tpu_address)
else:
return __retrieveWeights113(modelBuilder, weightsPath, tpu_address)
def createEntryCommentTable(self):
"""Create entry comment able.Existing table will be deleted.
"""
try:
transaction=self.__session.transaction()
transaction.start()
schema = self.__session.nominalSchema()
schema.dropIfExistsTable(CommonUtils.commentTableName())
description = coral.TableDescription()
description.setName(CommonUtils.commentTableName())
for columnName, columnType in self.__entryCommentTableColumns.items():
description.insertColumn(columnName,columnType)
for columnName in self.__entryCommentTableNotNullColumns:
description.setNotNullConstraint(columnName,True)
description.setPrimaryKey(self.__entryCommentTablePK)
tablehandle=schema.createTable(description)
tablehandle.privilegeManager().grantToPublic(coral.privilege_Select)
transaction.commit()
except Exception, e:
transaction.rollback()
raise Exception, str(e)
def existCommentTable(self):
"""Check if entry comment table exists
"""
try:
transaction=self.__session.transaction()
transaction.start(True)
schema = self.__session.nominalSchema()
result=schema.existsTable(CommonUtils.commentTableName())
transaction.commit()
#print result
except Exception, er:
transaction.rollback()
raise Exception, str(er)
def insertComment( self, tablename, entryid,comment ):
"""insert comment on the given entry of given table
"""
transaction=self.__session.transaction()
try:
transaction.start(False)
tabrowValueDict={'entryid':entryid,'tablename':tablename,'comment':comment}
schema = self.__session.nominalSchema()
dbop=DBImpl.DBImpl(schema)
dbop.insertOneRow(CommonUtils.commentTableName(),self.__entryCommentTableColumns,tabrowValueDict)
transaction.commit()
except Exception, e:
transaction.rollback()
raise Exception, str(e)
def insertSchedules(schedules):
insertSql = "insert into point(type,date,time,weekday,guest,home) values (?,?,?,?,?,?)"
insertValus = []
for point in schedules:
schedule = (point.type,point.date,point.time,point.weekday,point.guest,point.home)
insertValus.append(schedule)
conn = CommonUtils.getConnect()
cu = conn.cursor()
cu.execute("delete from point")
cu.executemany(insertSql,insertValus)
conn.commit()
conn.close()
def insertTeam(teams):
insertSql = "insert into team(eng_name,chinese_name,city,short_name) values(?,?,?,?)"
insertValues = []
for team in teams:
value = (team.engName, team.chineseName, team.city, team.shortName)
insertValues.append(value)
conn = CommonUtils.getConnect()
cu = conn.cursor()
cu.execute("delete from team")
cu.executemany(insertSql, insertValues)
conn.commit()
conn.close()
def bulkinsertComments( self, tableName,bulkinput):
"""bulk insert comments for a given table
bulkinput [{'entryid':unsigned long, 'tablename':string,'comment':string}]
"""
transaction=self.__session.transaction()
try:
transaction.start(False)
schema = self.__session.nominalSchema()
dbop=DBImpl.DBImpl(schema)
dbop.bulkInsert(CommonUtils.commentTableName(),self.__entryCommentTableColumns,bulkinput)
transaction.commit()
except Exception, e:
transaction.rollback()
raise Exception, str(e)
def clearAllEntriesForTable( self, tablename ):
"""delete all entries related with given table
"""
transaction=self.__session.transaction()
try:
transaction.start(False)
dbop=DBImpl.DBImpl(self.__session.nominalSchema())
condition='tablename = :tablename'
conditionbindDict=coral.AttributeList()
conditionbindDict.extend('tablename','string')
conditionbindDict['tablename'].setData(tablename)
dbop.deleteRows(CommonUtils.commentTableName(),condition,conditionbindDict)
transaction.commit()
except Exception, e:
transaction.rollback()
raise Exception, str(e)
def getTeamsInfo():
# base url
url = "http://g.hupu.com/nba/players/"
# teams
teams = [
"Spurs",
"Rockets",
"Mavericks",
"Grizzlies",
"Pelicans",
"Clippers",
"Warriors",
"Suns",
"Lakers",
"Kings",
"Blazers",
"Thunder",
"Nuggets",
"Timberwolves",
"Jazz",
"Raptors",
"Nets",
"Knicks",
"Celtics",
"76ers",
"Heat",
"Hawks",
"Wizards",
"Bobcats",
"Magic",
"Pacers",
"Bulls",
"Pistons",
"Cavaliers",
"Bucks",
]
# team dictionary
teamDic = {}
# get all html info divide by teamName
for team in teams:
teamUrl = url + team
teamPlayerInfos = CommonUtils.getSoupFromUrl(teamUrl)
teamDic[team] = teamPlayerInfos
return teamDic
def requestUrlContent(self, url, cache_dir=None, filename=None, js_enable=False, force_update=False):
if cache_dir != None and not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
ext = os.path.splitext(url)[-1]
if filename == None:
filename = CommonUtils.md5(url) + ext
target_path = None
if cache_dir == None:
target_path = tempfile.mktemp()
else:
target_path = os.path.join(".", cache_dir)
target_path = os.path.join(target_path, filename)
if target_path == None:
target_path = tempfile.mktemp()
wget = 'wget --user-agent="Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)"'
command = wget + ' "%s" -O %s --timeout=60 --tries=2'%(url, target_path)
if js_enable:
load_web_page_js_dir = os.path.join(".", "bin")
load_web_page_js = os.path.join(load_web_page_js_dir, "load.js")
# load_web_page_js = load_web_page_js.replace("\\", "/")
command = 'phantomjs --load-images=false "%s" "%s" "%s"'%(load_web_page_js, url, target_path)
# print "current dir", os.path.realpath(os.curdir)
# print "Request Url:", command.encode("utf-8")
if (not os.path.isfile(target_path)) or force_update:
state = os.system(command.encode("utf-8")) # 在windows下是gb2312, 在ubuntu主机上应该是utf-8
print url, "download successful!"
else:
# print url, "is already downloaded!"
pass
if ext[1:].lower() in PIC_SUFFIX:
return target_path
# 需要保证返回的字符串是ascii编码，否则lxml xpath解析可能会出问题
# phantomjs保存的文件是utf-8编码，所以需要进行转码为ascii编码
if js_enable:
return open(target_path).read().decode("utf8").encode("gb18030")
else:
return open(target_path).read()
def deleteCommentForId( self, tablename, entryid):
"""delete selected comment entry
"""
transaction=self.__session.transaction()
try:
transaction.start(False)
dbop=DBImpl.DBImpl(self.__session.nominalSchema())
condition='tablename = :tablename AND entryid = :entryid'
conditionbindDict=coral.AttributeList()
conditionbindDict.extend('tablename','string')
conditionbindDict.extend('entryid','unsigned long')
conditionbindDict['tablename'].setData(tablename)
conditionbindDict['entryid'].setData(entryid)
dbop.deleteRows(CommonUtils.commentTableName(),condition,conditionbindDict)
transaction.commit()
except Exception, e:
transaction.rollback()
raise Exception, str(e)
def getTeamUrlContent():
teamUrl = "http://g.hupu.com/nba/teams/"
#teams
teams = ['Spurs','Rockets','Mavericks','Grizzlies','Pelicans','Clippers',
'Warriors','Suns','Lakers','Kings','Blazers','Thunder','Nuggets',
'Timberwolves','Jazz','Raptors','Nets','Knicks','Celtics','76ers',
'Heat','Hawks','Wizards','Bobcats','Magic','Pacers','Bulls',
'Pistons','Cavaliers','Bucks']
teamDic = {}
for teamName in teams:
url = teamUrl + teamName
teamInfo = CommonUtils.getSoupFromUrl(url)
teamDic[teamName] = teamInfo
return teamDic
def replaceId( self, tableName, oldentryid, newentryid ):
"""replace entryid in given table
"""
transaction=self.__session.transaction()
try:
transaction.start(False)
editor = self.__session.nominalSchema().tableHandle(CommonUtils.commentTableName()).dataEditor()
inputData = coral.AttributeList()
inputData.extend('newentryid','unsigned long')
inputData.extend('oldentryid','unsigned long')
inputData.extend('tablename','string')
inputData['newentryid'].setData(newentryid)
inputData['oldentryid'].setData(oldentryid)
inputData['tablename'].setData(tableName)
editor.updateRows( "entryid = :newentryid", "entryid = :oldentryid AND tablename = :tablename", inputData )
transaction.commit()
except Exception, e:
transaction.rollback()
raise Exception, str(e)
def requestUrlContent(self, url, cache_dir=None, filename=None):
if cache_dir != None and not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
ext = os.path.splitext(url)[-1]
if filename == None:
filename = CommonUtils.md5(url) + ext
target_path = None
if cache_dir == None:
target_path = tempfile.mktemp()
else:
target_path = os.path.join(".", cache_dir)
target_path = os.path.join(target_path, filename)
if target_path == None:
target_path = tempfile.mktemp()
command = 'wget "%s" -O %s '%(url, target_path)
print "Request Url:", command
state = 0
if not os.path.isfile(target_path):
# 在windows下是gb2312, 在ubuntu主机上应该是utf-8
if platform.system() == 'Windows':
state = os.system(command.encode("gb2312"))
else:
state = os.system(command.encode("utf-8"))
else:
print url, "is already downloaded!"
if ext[1:].lower() in PIC_SUFFIX:
# print "requestUrlContent state", state
if state != 0:
return None
return target_path
return open(target_path).read()
if platform.system() == 'Windows':
host="localhost"
user="******"
passwd="123456"
db="test"
deploy = AWSBaiduNewsDeploy(host, user, passwd, db)
deploy.start()
if __name__ == "__main__":
reload(sys)
sys.setdefaultencoding('utf-8')
workDir = os.path.dirname(os.path.realpath(sys.argv[0]))
os.chdir(os.path.dirname(workDir))
logFile = CommonUtils.openLogFile()
oldStdout = sys.stdout
sys.stdout = logFile
print "============================================"
print "change work direcotory to workDir", workDir
print "Start AWS Baidu News Deploy:", time.asctime()
main()
logFile.close()
if oldStdout:
sys.stdout = oldStdout
def run(self):
line = self.stream.readline()
while line:
try:
line = line.strip()
if len(line) == 0:
continue
qTuple = line.split()
if len(qTuple) <> 9:
self.errList.append("Wrong partition info column number: %d" % len(qTuple))
continue
queue = qTuple[0]
if queue.endswith('*'):
queue = queue[:-1]
if not queue in self.qtable:
self.qtable[queue] = PartitionInfo()
if qTuple[1] == 'down' or qTuple[1] == 'inactive':
self.qtable[queue].state = 'Closed'
elif qTuple[1] == 'drain':
self.qtable[queue].state = 'Draining'
else:
self.qtable[queue].state = 'Production'
parsed = self.cpuRegex.match(qTuple[2])
if not parsed:
self.errList.append("Wrong format for partition cpu info: " + qTuple[2])
continue
self.qtable[queue].freeCPU = int(parsed.group(2))
self.qtable[queue].activeCPU = int(parsed.group(1))
self.qtable[queue].totalCPU = int(parsed.group(4))
if qTuple[3] <> 'n/a':
self.qtable[queue].maxRuntime = CommonUtils.convertTimeLimit(qTuple[3])
if qTuple[4] <> 'n/a':
self.qtable[queue].defaultRuntime = CommonUtils.convertTimeLimit(qTuple[4])
elif self.qtable[queue].maxRuntime <> -1:
self.qtable[queue].defaultRuntime = self.qtable[queue].maxRuntime
try:
minNodes, maxNodes = CommonUtils.convertJobSize(qTuple[5])
if maxNodes < 0:
continue
if qTuple[7].lower() <> 'unlimited':
maxCPUNode = int(qTuple[7])
self.qtable[queue].slotsPerJob = maxNodes * maxCPUNode
else:
tmpl = [ i.translate(None, '+') for i in qTuple[8].split(':') ]
socketNum = int(tmpl[0])
coreNum = int(tmpl[1])
thrNum = int(tmpl[2])
self.qtable[queue].slotsPerJob = maxNodes * socketNum * coreNum * thrNum
except Exception, ex:
logger.debug("Cannot calculate MaxSlotsPerJob for %s", queue, exc_info=True)
self.errList.append("Cannot calculate MaxSlotsPerJob for %s" % queue)
finally:
line = self.stream.readline()
def init(): scheduleUrl = "http://nba.sports.sina.com.cn/match_result.php?day=all" return CommonUtils.getSoupFromUrl(scheduleUrl)
db=JDConstants.MYSQL_DATABASE
spider = JDProductSpider(host, user, passwd, db)
spider.start()
# product = JDProductInfo()
# product.setProductId("1453838631")
# product.setProductBuyUrl("http://item.jd.com/1453838631.html")
# spider.getProductDetail(product)
if __name__=="__main__":
reload(sys)
sys.setdefaultencoding('utf-8')
workDir = os.path.dirname(os.path.realpath(sys.argv[0]))
os.chdir(os.path.dirname(workDir)) # 保证spider cache目录一致
logFile = CommonUtils.openLogFile(mode="w")
oldStdout = sys.stdout
sys.stdout = logFile
print "============================================"
print "change work direcotory to workDir", workDir
print "Start Baidu Product Spider:", time.asctime()
main()
logFile.close()
if oldStdout:
sys.stdout = oldStdout
def getUrlContent():
return CommonUtils.getSoupFromUrl("http://g.hupu.com/nba/daily/boxscore_23325.html")
def run(self):
line = self.stream.readline()
while line:
try:
jTable = dict()
parsed = self.jstateRegex.search(line)
if not parsed:
continue
if parsed.group(1) == "RUNNING":
jTable['state'] = 'running'
else:
jTable['state'] = 'queued'
parsed = self.jidRegex.search(line)
if not parsed:
continue
jTable['jobid'] = parsed.group(1)
parsed = self.nameRegex.search(line)
if not parsed:
continue
jTable['name'] = parsed.group(1)
parsed = self.uidRegex.search(line)
if not parsed:
continue
jTable['user'] = parsed.group(1)
parsed = self.gidRegex.search(line)
if not parsed:
continue
jTable['group'] = parsed.group(1)
parsed = self.partRegex.search(line)
if not parsed:
continue
jTable['queue'] = parsed.group(1)
parsed = self.ncpuRegex.search(line)
if not parsed:
continue
jTable['cpucount'] = int(parsed.group(1))
parsed = self.tlimitRegex.search(line)
if parsed:
tmpLimit = parsed.group(1)
if tmpLimit <> 'UNLIMITED':
jTable['maxwalltime'] = CommonUtils.convertTimeLimit(tmpLimit)
parsed = self.subtimeRegex.search(line)
if not parsed:
continue
else:
jTable['qtime'] = self.convertTime(parsed.group(1))
parsed = self.sttimeRegex.search(line)
if parsed:
jTable['start'] = self.convertTime(parsed.group(1))
self.jobTables.append(jTable)
finally:
line = self.stream.readline()
