def writeToUSB(self,fileName):
r'''
把处理好的帧数据连续写入USB设备；
'''
try:
mylog.info('正在写入帧文件...')
frameData = numpy.load('./files/%s.npy' % fileName)
#某些情况下需要降低发送速度，下面将帧数据分块发送；
frameData = bytes(frameData)
dataLength = len(frameData)
bytesValidOneTime = int(0.5*1024) #每次发送的数据中，有效帧的字节数
i = 0
while (i+bytesValidOneTime) < dataLength:
#time.sleep(0.001)
if not self.usbHandler.writeToUSB(frameData[i:(i+bytesValidOneTime)]):
mylog.error('Error: 写入失败！')
return False
i += bytesValidOneTime
if i < dataLength:
if not self.usbHandler.writeToUSB(frameData[i:]):
mylog.error('Error: 写入失败！')
return False
except BaseException as e:
mylog.error("Error: 写入帧文件失败。")
mylog.error(str(e))
return False
else:
mylog.info('已成功写入帧文件。')
return True
def deregister(self, usernames: List[str]):
with self.lock:
if tuple(usernames) in self.streams:
s = self.streams[tuple(usernames)]
del self.streams[tuple(usernames)]
s.disconnect()
else:
# Technically, we don't have any timing guarantees about
# when run_short_poll can prune the list down to size 2.
# Practically, if we hit this limit, something has gone very wrong.
assert len(self.short_poll) < 100, self.short_poll
# Split list based on predicate "Should it be removed?"
users_pruned = []
users_passed = []
for e in self.short_poll:
l = users_pruned if e['user'] in usernames else users_passed
l.append(e)
if len(users_pruned) != len(usernames):
mylog.warning(
"Tried to remove nonexistent usernames entries.")
mylog.warning(" Actually pruned: {}", users_pruned)
mylog.warning(" Wanted to prune: {}", usernames)
if len(users_pruned) > 0:
mylog.info(
"Some short-polls removed: {}".format(users_pruned))
self.short_poll = users_passed
for e in users_pruned:
self.consumer_updates.updateShortpoll(
e['name'], 'del-timeout')
return
def start_app(self,startpath,updatepath,command):
'''起程序'''
# startpath = self.startpath
# updatepath = self.updatepath
apppath = os.path.split(startpath)[0]
apppname = os.path.split(startpath)[1]
# command = self.command
#print startpath
if os.path.isfile(startpath):
try:
os.chdir(apppath)#如果启动程序不在第一层目录下？ 需优化
# #os.system(startpath)#启动程序后应该退出脚本，现没有退出 一直占用资源 需优化【用start app或 execl解决】
# os.system('start %s' %apppname)
# #os.execl(startpath,'i') #直接退出进程
# #规避删除目录error32错误，临时解决方法
#os.chdir(updatepath)
#bug3 os.system执行后进程不带路径导致close失败
if command is None:
subprocess.Popen(startpath)
else:
subprocess.Popen([command,startpath])
#bug1 规避删除目录error32错误，临时解决方法
os.chdir(updatepath)
mylog.info('启动服务成功')
return True
# subprocess.Popen(startpath, shell=True)
except Exception,msg:
mylog.error('启动服务失败，%s'%msg)
return False
def deregister(self, usernames: List[str]):
with self.lock:
if tuple(usernames) not in self.user_blocks:
mylog.warning("Tried to remove nonexistent usernames entry {}".format(usernames))
return
self.user_blocks.remove(tuple(usernames))
mylog.info("Successfully removed {}".format(usernames))
def test_batching_x(n, batch):
if not RUN_SLOW_TESTS:
mylog.warning("[SKIP] Skipping slow test")
return
mylog.info("Testing batching.TweetBatcher:")
conn = mq.PrintQueue.new('test_batching')
batcher = mq.Batcher(conn)
expected = []
for i in range(n):
s = "Should be batch {batch}, message {i}/{n}" \
.format(batch=batch, i=i, n=n)
expected.append(s)
batcher.post(s)
time.sleep(mq.BATCH_TIMEOUT / 2.0)
if hasattr(conn, 'expect'):
conn.expect([])
time.sleep(mq.BATCH_TIMEOUT)
if hasattr(conn, 'expect'):
# Expect precisely one message with all "parts" bundled up.
conn.expect([expected])
expected = 'Check second run'
batcher.post(expected)
time.sleep(mq.BATCH_TIMEOUT / 2.0)
if hasattr(conn, 'expect'):
conn.expect([])
time.sleep(mq.BATCH_TIMEOUT)
if hasattr(conn, 'expect'):
batch = [expected]
all_batches = [batch]
conn.expect(all_batches)
def deregister(self, usernames: List[str]):
with self.lock:
if tuple(usernames) in self.streams:
s = self.streams[tuple(usernames)]
del self.streams[tuple(usernames)]
s.disconnect()
else:
# Technically, we don't have any timing guarantees about
# when run_short_poll can prune the list down to size 2.
# Practically, if we hit this limit, something has gone very wrong.
assert len(self.short_poll) < 100, self.short_poll
# Split list based on predicate "Should it be removed?"
users_pruned = []
users_passed = []
for e in self.short_poll:
l = users_pruned if e['user'] in usernames else users_passed
l.append(e)
if len(users_pruned) != len(usernames):
mylog.warning("Tried to remove nonexistent usernames entries.")
mylog.warning(" Actually pruned: {}", users_pruned)
mylog.warning(" Wanted to prune: {}", usernames)
if len(users_pruned) > 0:
mylog.info("Some short-polls removed: {}".format(users_pruned))
self.short_poll = users_passed
for e in users_pruned:
self.consumer_updates.updateShortpoll(e['name'], 'del-timeout')
return
def zip_dir(self,copypath,testpath):
'''打包备份'''
# copypath = self.copypath
# testpath = self.testpath
nowtime = time.strftime("%Y%m%d%H%M%S", time.localtime())
if os.path.isdir(copypath):
zipFile1 = zipfile.ZipFile(os.path.join(copypath,os.path.basename(testpath)+nowtime+'.zip'),'w')
else:
mylog.error('备份失败，备份目录不合法')
return False
#用宽度优先搜索全路径
if os.path.isdir(testpath):
filelist = os.listdir(testpath)
while len(filelist)>0:
out = filelist.pop()
if os.path.isdir(os.path.join(testpath,out)):
#os.path.join(os.path.basename(testpath),out) 短目录替换全路径
#out 当前路径
zipFile1.write(os.path.join(testpath,out),out)
for outfile in os.listdir(os.path.join(testpath,out)):
filelist.append(os.path.join(out,outfile))
else:
zipFile1.write(os.path.join(testpath,out),out)
else:
mylog.error('备份失败，测试目录不合法')
return False
#用os.walk搜索全路径下文件,还不能做到压缩目录不带全路径
"""for dirpath, dirnames, filenames in os.walk(testpath):
for filename in filenames:
#print filename
zipFile1.write(os.path.join(dirpath,filename))"""
zipFile1.close()
mylog.info('备份成功')
return True
def dequeue_cmd_q(self):
mylog.info("cmd_registry.dequeue_cmd_q()")
try:
cmd_str = self.cmd_q.get_nowait()
self.parse_cmd(cmd_str)
except:
pass
def createOrCached(src_path, dst_path, length_ms):
if os.path.exists(dst_path):
mylog.info("soundGenerator: using cached file: " + dst_path)
# no-op
return length_ms
else:
mylog.info("soundGenerator: creating new file: " + dst_path)
return createNewSoundfile(src_path, dst_path, length_ms)
def add_cmd(self, cmd):
mylog.info("cmd_registry.add_cmd>cmd=>" + str(cmd))
for c in self.cmds:
if c.cmd_type == cmd.cmd_type:
raise Exception("Command Type already registered: " +
str(cmd.cmd_type))
self.cmds.append(cmd)
def deregister(self, usernames: List[str]):
with self.lock:
if tuple(usernames) not in self.user_blocks:
mylog.warning(
"Tried to remove nonexistent usernames entry {}".format(
usernames))
return
self.user_blocks.remove(tuple(usernames))
mylog.info("Successfully removed {}".format(usernames))
def on_status(self, status):
tweet = parse_tweet_status(status)
if tweet is None:
mylog.error("{}: on_tweet BROKEN! (skip)".format(self.desc))
elif tweet['uid'] not in self.sensitive:
mylog.info("{}: dropped irrelevant tweet from user {} at time {}"
.format(self.desc, tweet['uid'], tweet['time']))
else:
self.on_tweet(tweet)
def copytjb():
"""拷文件"""
mylog.info('准备复制olddir，reportfile')
try:
shell.SHFileOperation ((0, shellcon.FO_COPY, olddir, newdir,shellcon.FOF_NOCONFIRMMKDIR|shellcon.FOF_NOCONFIRMATION, None, None))
shell.SHFileOperation ((0, shellcon.FO_COPY, reportfile, localreportfile,shellcon.FOF_NOCONFIRMATION, None, None))
shell.SHFileOperation ((0, shellcon.FO_COPY, svnfile, localsvnfile,shellcon.FOF_NOCONFIRMATION, None, None))
mylog.info('复制olddir，reportfile成功')
except:
mylog.error('复制失败，请关闭相关程序，再试')
def maybe_reply(self, tweet_id: str, content: str):
mylog.info("-" * 40)
mylog.info("About to respond ({} chars): {}".format(len(content), content))
if self.key not in MAY_POST:
mylog.info("Not posting this reply, as key {} is not permitted. Try one of {} instead."
.format(self.key, MAY_POST))
else:
mylog.info("Actually writing to Twitter!")
self.api.update_status(status=content, in_reply_to_status_id=tweet_id)
mylog.info("-" * 40)
def on_status(self, status):
tweet = parse_tweet_status(status)
if tweet is None:
mylog.error("{}: on_tweet BROKEN! (skip)".format(self.desc))
elif tweet['uid'] not in self.sensitive:
mylog.info(
"{}: dropped irrelevant tweet from user {} at time {}".format(
self.desc, tweet['uid'], tweet['time']))
else:
self.on_tweet(tweet)
def test_responses():
responses = []
responses.extend(responseBuilder.build_worst_acks())
responses.extend(responseBuilder.build_worst_nacks())
# Not a dict to preserve order
lengths = [(len(r), r) for r in responses]
mylog.info("Worst-case response lengths: {}".format(
[len(r) for r in responses]))
for (l, r) in lengths:
assert l <= MAX_RESPONSE_LENGTH, (l, r, MAX_RESPONSE_LENGTH)
def findUSB(self):
self.device = []
self.deviceW = None
self.deviceR = None
self.inPoint = None
self.outPoint = None
self.usbReady = False
self.device = list(usb.core.find(find_all=True))
self.deviceOrder[1], self.deviceOrder[0] = self.deviceOrder
usbDeviceNum = len(self.device)
if usbDeviceNum == 1:
self.deviceR = self.device[0]
self.deviceR.set_configuration()
cfgR = self.deviceR.get_active_configuration()
intfR = cfgR[(0, 0)]
self.inPoint = usb.util.find_descriptor(intfR,
# match the first IN endpoint
custom_match = lambda e: \
usb.util.endpoint_direction(e.bEndpointAddress) == \
usb.util.ENDPOINT_IN)
self.usbReady = True
mylog.info("1个USB设备已连接。")
elif usbDeviceNum == 2:
self.deviceW = self.device[self.deviceOrder[0]]
self.deviceR = self.device[self.deviceOrder[1]]
self.deviceW.set_configuration()
cfgW = self.deviceW.get_active_configuration()
intfW = cfgW[(0, 0)]
self.deviceR.set_configuration()
cfgR = self.deviceR.get_active_configuration()
intfR = cfgR[(0, 0)]
self.outPoint = usb.util.find_descriptor(intfW,
# match the first OUT endpoint
custom_match = lambda e: \
usb.util.endpoint_direction(e.bEndpointAddress) == \
usb.util.ENDPOINT_OUT)
self.inPoint = usb.util.find_descriptor(intfR,
# match the first IN endpoint
custom_match = lambda e: \
usb.util.endpoint_direction(e.bEndpointAddress) == \
usb.util.ENDPOINT_IN)
self.usbReady = True
mylog.info("2个USB设备已连接。")
else:
self.usbReady = False
mylog.error("Error: 可用USB设备数量不是2。")
def slot_exists(self):
mylog.info("parkinglot.slot_exists()")
self.has_parking()
for index in range(self.n_lots):
if self.lots[index] == None:
mylog.info("[RETURN][index]=" + str(index))
return index
raise Exception("Sorry, parking lot is full")
def findstrinfile(filename, lookup):
"""通过report判断集成是否成功"""
if lookup not in open(filename,'rt').read():
mylog.info('集成操作成功')
return True
else:
mylog.info('版本集成失败，请通知相关人员，不做操作')
root = Tkinter.Tk()
label =Tkinter.Label(root,text='集成失败，请通知相关人员').pack()
root.geometry("230x30+600+300")
root.mainloop()
return False
def __init__(self):
self.polByPid = json.load(open(BACKEND_POLI_DB))
self.lock = threading.RLock()
# This references the same politician dicts, just like pointers.
# In other words: updates will always be reflected in both lookup tables.
self.polByTid = dict()
for poli in self.polByPid.values():
assert "twittering" in poli, poli['pid']
self.polByTid[str(poli["twittering"]["twitterId"])] = poli
mylog.info("Loaded {} polititians; {} of them have a TID"
.format(len(self.polByPid), len(self.polByTid)))
def maybe_reply(self, tweet_id: str, content: str):
mylog.info("-" * 40)
mylog.info("About to respond ({} chars): {}".format(
len(content), content))
if self.key not in MAY_POST:
mylog.info(
"Not posting this reply, as key {} is not permitted. Try one of {} instead."
.format(self.key, MAY_POST))
else:
mylog.info("Actually writing to Twitter!")
self.api.update_status(status=content,
in_reply_to_status_id=tweet_id)
mylog.info("-" * 40)
def do_update(self,updatepath):
"""从svn更新最新程序(不管有没更新都会进行后续操作，需优化)"""
# updatepath = self.updatepath
#检查路径是否为目录
if os.path.isdir(updatepath):
try:
updateshell = 'TortoiseProc.exe /command:update /path:"%s" /closeonend:1'%updatepath
os.system(updateshell)
mylog.info('升级成功')
return True
except Exception,msg:
mylog.error('升级失败，%s'%msg)
return False
def parse_cmd(self, cmd_str):
mylog.info("cmd_registry.parse_cmd")
mylog.info("[cmd_str]=" + cmd_str)
found = False
for cmd in self.cmds:
if cmd.parse(cmd_str) == True:
self.parse_cmd_(cmd)
found = True
break
if not found:
self.add_output("Invalid Command : " + cmd_str)
def park_car(self, car, index):
mylog.info("parkinglot.park_car()")
mylog.info("[car]=" + str(car))
mylog.info("[index]=" + str(index))
self.lots[index] = car
mylog.info("[RETURN][index]=" + str(index + 1))
return (index + 1)
def test_messages_send():
mylog.info("Testing messages.UpdatesQueueAdapter:")
conn = mq.PrintQueue.new('test_msg_send')
adapter = messages.UpdatesQueueAdapter(conn)
adapter.updateShortpoll('karli', 'succ-firstpoll')
expected = {
'twittername': 'karli',
'status': 'succ',
'reason': 'succ-firstpoll',
'message': {
"de": 'karli kann los-tweeten!',
"en": 'karli can start tweeting!'
}
}
conn.expect([expected])
def test_all():
# This might show weird behavior if you modify MANUAL_TESTS by hand
mylog.info('[TEST] -- Running all tests --')
for t in all_tests:
mylog.info("[TEST] {}".format(t))
t()
mylog.info("[DONE] {}".format(t))
mylog.info('[DONE] -- Done with all tests --')
def handle(self,conn):
r = redis.Redis(host='10.4.6.39',port=6379,db=0)
req = urllib2.Request('http://bi.jtjr99.com/export/appstat')
response = urllib2.urlopen(req)
data = response.read()
js = json.loads(data)
total_customer = long(js['data']['allusers'])
total_amount = long(js['data']['tx_amount_all'])
r.set('total_customer',total_customer)
r.set('total_amount',total_amount)
msg = 'total_customer:%s,total_amount:%s' % (total_customer,total_amount)
mylog.info(msg)
def test_messages_phrasing():
mylog.info("Testing messages.phrase:")
expected = {
'twittername': 'heinzelmännchen',
'status': 'fail',
'reason': 'unknown-user',
'message': {
"de": 'Konnte "heinzelmännchen" nicht auf Twitter finden.',
"en": 'Could not find "heinzelmännchen" on twitter.'
}
}
actual = messages.phrase("heinzelmännchen", 'unknown-user')
assert actual == expected, (actual, expected)
mylog.warning(
"[MANU] Testing mq.RealQueue. Check http://localhost:15672/#/queues/%2F/test_mq"
)
def test_mq():
def run_with(maker):
conn = maker("test_mq")
if hasattr(conn, 'expect'):
conn.expect([])
for s in ["My lovely message", "Ümläöts Partyß! Or … what¿"]:
conn.post(s)
if hasattr(conn, 'expect'):
conn.expect([s])
mylog.info("Testing mq.PrintQueue:")
run_with(mq.PrintQueue.new)
mylog.warning(
"[MANU] Testing mq.RealQueue. Check http://localhost:15672/#/queues/%2F/test_mq"
)
run_with(mq.RealQueue.new)
def execute(self, cmd):
"""
CREATE_PARKING = "create_parking_lot"
PARK = "park"
LEAVE = "leave"
STATUS = "status"
PRINT_REGISTERNO_WCOLOR = "registration_numbers_for_cars_with_colour"
PRINT_SLOTNO_WCOLOR = "slot_numbers_for_cars_with_colour"
PRINT_SLOTNO_WREGISTERNO = "slot_numbers_for_registration_number"
"""
mylog.info("parking_lot.execute()")
msg = ""
try:
if cmd.cmd_type == cmd_type.CREATE_PARKING:
if self.create_slots(cmd.argument[0]):
msg = "Created a parking lot with " + str(
cmd.argument[0]) + " slots"
elif cmd.cmd_type == cmd_type.PARK:
index = self.park(cmd.argument[0], cmd.argument[1])
if index > 0:
msg = "Allocated slot number: " + str(index)
elif cmd.cmd_type == cmd_type.LEAVE:
if self.leave(cmd.argument[0]):
msg = "Slot number " + str(cmd.argument[0]) + " is free"
elif cmd.cmd_type == cmd_type.STATUS:
msg = self.status()
elif cmd.cmd_type == cmd_type.PRINT_REGISTERNO_WCOLOR:
msg = self.print_registration_no_car_wcolor(cmd.argument[0])
elif cmd.cmd_type == cmd_type.PRINT_SLOTNO_WCOLOR:
msg = self.print_slot_no_car_wcolor(cmd.argument[0])
elif cmd.cmd_type == cmd_type.PRINT_SLOTNO_WREGISTERNO:
msg = self.print_slot_no_car_wregister(cmd.argument[0])
except Exception as ex:
msg = str(ex)
mylog.info("[Return][msg]=" + msg)
return msg + "\n"
def run(self):
self.running = True
mylog.info('cmd_out thread started')
while self.running == True:
try:
output = self.cmd_registry.output.get_nowait()
if output == "exit":
self.running = False
if self.on_exit:
self.on_exit()
else:
print output
except:
pass
#print "cmd_out.running=true"
time.sleep(0.1)
def handle_poli(self, tweet, msg, poli):
# Careful: 'poli' is a copy, so any changes due to setBird aren't reflected!
msg['partycolor'] = party_to_color(poli['party'])
msg['party'] = poli['party']
pBird = poli['self_bird']
# In case it changed, use the one provided by twitter
handle = msg['twitterName']
has_command = contains_command(tweet['hashtags'])
# Check for any updates
if 'house' in tweet['username'].lower(
) and tweet['content'].startswith('@'):
mylog.warning(
"Ignoring my own tweet for commands, as it starts with '@'")
elif has_command:
pid = poli['pid']
pBird_name = self.birdBack.getName(pBird)
bird_id = find_bird(tweet['content'], self.birdBack)
reply = None
if bird_id is not None:
# Ack
bird_name = self.birdBack.getName(bird_id)
mylog.info('politician "{}" ({}) gets new bird {}'.format(
tweet['userscreen'], pid, bird_id))
msg['refresh'] = dict()
msg['refresh']['politicianId'] = pid
msg['refresh']['birdId'] = bird_id
self.pb.setBird(pid, bird_id, actor='p')
reply = responseBuilder.build_some_ack(handle, pBird_name,
bird_name)
# Again, 'poli' is a copy, so it wasn't updated by the call to 'setBird'.
pBird = bird_id
elif has_command != COMMAND_HASHTAGS_ACKONLY:
# NACK
mylog.warning('I saw that command, but no valid bird!')
mylog.warning('pid={pid!r} content={ct}'.format(
ct=tweet['content'], pid=pid))
reply = responseBuilder.build_some_nack(handle, pBird_name)
if reply is not None:
self.tw.twitter.maybe_reply(tweet['tweet_id'], reply)
# In case of 'refresh', poli already contains the update:
return [poli['citizen_bird'], pBird]
def addCitizen(self, twittername, birdid, tid=None):
if tid is None:
tid = self.twitter.resolve_name(twittername)
if tid is None:
mylog.warning("citizen user ignored, invalid name: " + twittername)
self.consumer_updates.updateShortpoll(twittername, "unknown-user")
return
if self.polBack.getPolitician(tid) is not None:
self.consumer_updates.updateShortpoll(twittername, "is-politician")
return
if birdid not in self.birdBack.bJson:
mylog.warning("citizen user ignored, invalid bird: " + birdid)
self.consumer_updates.updateShortpoll(twittername, "unknown-bird")
return
with self.lock:
if tid in self.citizens:
entry = self.citizens[tid]
mylog.info(
"Updating existing citizen's bird from {}".format(entry))
else:
mylog.info("Creating new citizen's bird")
entry = dict()
entry["userId"] = tid
entry["party"] = 'neutral'
self.citizens[tid] = entry
# Even if a tweet comes in instantly, getCitizen syncs on
# self.lock, so it's fine. That's also why getCitizen() will
# never see an incomplete citizen.
self.twitter.register_shortlived(tid, twittername)
entry["birdId"] = birdid
token = poll_counter()
entry["token"] = token
mylog.debug("Resulting citizen entry: {}".format(entry))
timer = threading.Timer(REMOVE_CITIZEN_TIME,
self._remove_citizen_wrap, [tid, token])
# Don't prevent shutting down
timer.daemon = True
timer.start()
self.consumer_updates.updateShortpoll(twittername, "succ-resolved")
return
def addCitizen(self, twittername, birdid, tid=None):
if tid is None:
tid = self.twitter.resolve_name(twittername)
if tid is None:
mylog.warning("citizen user ignored, invalid name: " + twittername)
self.consumer_updates.updateShortpoll(twittername, "unknown-user")
return
if self.polBack.getPolitician(tid) is not None:
self.consumer_updates.updateShortpoll(twittername, "is-politician")
return
if birdid not in self.birdBack.bJson:
mylog.warning("citizen user ignored, invalid bird: " + birdid)
self.consumer_updates.updateShortpoll(twittername, "unknown-bird")
return
with self.lock:
if tid in self.citizens:
entry = self.citizens[tid]
mylog.info("Updating existing citizen's bird from {}".format(entry))
else:
mylog.info("Creating new citizen's bird")
entry = dict()
entry["userId"] = tid
entry["party"] = 'neutral'
self.citizens[tid] = entry
# Even if a tweet comes in instantly, getCitizen syncs on
# self.lock, so it's fine. That's also why getCitizen() will
# never see an incomplete citizen.
self.twitter.register_shortlived(tid, twittername)
entry["birdId"] = birdid
token = poll_counter()
entry["token"] = token
mylog.debug("Resulting citizen entry: {}".format(entry))
timer = threading.Timer(REMOVE_CITIZEN_TIME,
self._remove_citizen_wrap, [tid, token])
# Don't prevent shutting down
timer.daemon = True
timer.start()
self.consumer_updates.updateShortpoll(twittername, "succ-resolved")
return
def test_name_resolution():
ids = {
'HouseOfTweetsSB': '4718199753',
'@HouseOfTweetsSB': '4718199753',
'@MissesVlog': '50712079',
'eeQu0Ae4': '774336282101178368',
'SarcasticTester': '720224609560371201',
'HopefullyNotARealTwitterAccount': None,
'thisismyspacesofkoff': None,
'@thisismyspacesofkoff': None,
'@nowgetinside': '776147921146445824',
'nowgetinside': '776147921146445824',
}
twi = twitter.RealTwitterInterface()
for (user, expect_id) in ids.items():
actual_id = twi.resolve_name(user)
mylog.info('resolve {u} to {a!r} (expected {e!r})'.format(u=user,
e=expect_id,
a=actual_id))
assert actual_id == expect_id
def _remove_citizen(self, tid, token):
with self.lock:
mylog.info("Want to remove citizen {}, token {}".format(tid, token))
if tid not in self.citizens:
mylog.warning("=> Already deleted (huh?)")
elif self.citizens[tid]['token'] != token:
mylog.info("=> Token mismatch, db has {}"
.format(self.citizens[tid]['token']))
else:
mylog.info("=> Yup")
self.twitter.deregister([tid])
del self.citizens[tid]
mylog.info("Remaining citizens: {}".format(self.citizens.keys()))
def find_pair(bird: Union[None, str], mood: str, retweet: bool, length: int):
if bird is None:
return None
candidates = [(bird, mood, retweet),
(bird, 'neutral', retweet),
(bird, 'fragend', retweet),
(bird, 'aufgebracht', retweet),
('amsel', 'neutral', False)]
verbose = False
for (b, m, r) in candidates:
candidSource = path_raw(b, m, r)
if os.path.isfile(candidSource):
if verbose:
mylog.info("Found at {}".format(candidSource))
return candidSource, path_processed(b, m, r, length)
else:
mylog.warning("Source file {} missing, falling back ...".format(candidSource))
verbose = True
mylog.error("All sources and fallbacks missing. Giving up.")
return None
def close_app(self,startpath):
'''关闭程序进程'''
# startpath = self.startpath
# try:
# closeshell = 'taskkill /f /im %s'%startpath
# os.system(closeshell)
# except Exception,msg:
# print msg
# return False
startpath = self.startpath
# print startpath
pid = self.check_process(startpath)
if pid:
try:
psutil.Process(pid).kill()
mylog.info('关闭服务成功,进程号为%s'%pid)
return True
except Exception,msg:
mylog.error('关闭服务失败，%s'%msg)
return False
def post(self, message, isRetry=False):
mylog.info('Publishing on queue {name}: {data!r}'
.format(name=self.name, data=message))
with self.lock:
if self.connection.is_closed:
mylog.warning("Whoops, connection is closed; reopen.")
self.connection = connection()
self.channel = self.connection.channel()
try:
self.channel.basic_publish(exchange='', routing_key=self.name,
body=json.dumps(message))
self.maybe_log_message(message)
except Exception as e:
mylog.error("Connection failed anyway? Make sure RabbitMQ is running! (is_closed = {})"
.format(self.connection.is_closed))
mylog.error(e.__repr__())
mylog.error(e.args)
mylog.info("Message dropped.")
if not isRetry:
self.post(message, True)
def handle_poli(self, tweet, msg, poli):
# Careful: 'poli' is a copy, so any changes due to setBird aren't reflected!
msg['partycolor'] = party_to_color(poli['party'])
msg['party'] = poli['party']
pBird = poli['self_bird']
# In case it changed, use the one provided by twitter
handle = msg['twitterName']
has_command = contains_command(tweet['hashtags'])
# Check for any updates
if 'house' in tweet['username'].lower() and tweet['content'].startswith('@'):
mylog.warning("Ignoring my own tweet for commands, as it starts with '@'")
elif has_command:
pid = poli['pid']
pBird_name = self.birdBack.getName(pBird)
bird_id = find_bird(tweet['content'], self.birdBack)
reply = None
if bird_id is not None:
# Ack
bird_name = self.birdBack.getName(bird_id)
mylog.info('politician "{}" ({}) gets new bird {}'
.format(tweet['userscreen'], pid, bird_id))
msg['refresh'] = dict()
msg['refresh']['politicianId'] = pid
msg['refresh']['birdId'] = bird_id
self.pb.setBird(pid, bird_id, actor='p')
reply = responseBuilder.build_some_ack(handle, pBird_name, bird_name)
# Again, 'poli' is a copy, so it wasn't updated by the call to 'setBird'.
pBird = bird_id
elif has_command != COMMAND_HASHTAGS_ACKONLY:
# NACK
mylog.warning('I saw that command, but no valid bird!')
mylog.warning('pid={pid!r} content={ct}'
.format(ct=tweet['content'], pid=pid))
reply = responseBuilder.build_some_nack(handle, pBird_name)
if reply is not None:
self.tw.twitter.maybe_reply(tweet['tweet_id'], reply)
# In case of 'refresh', poli already contains the update:
return [poli['citizen_bird'], pBird]
def run(self):
self.running = True
mylog.info("cmd_registry thread started.")
while self.running == True:
try:
cmd_str = self.cmd_q.get_nowait()
if cmd_str == "exit":
self.running = False
if self.on_exit:
self.on_exit()
else:
self.parse_cmd(cmd_str)
except Queue.Empty:
pass
except Exception as ex:
self.add_output(str(ex))
#print "cmd_registry.running=true"
time.sleep(0.1)
def transformFrameFile(self,fileName):
r'''
把64bit的TXT帧文件转成bytes类型的.npy文件，用于传输；
'''
#读入40比特的帧数据
mylog.info('正在将64bit帧文件转换为64bit的倒序帧文件...')
with open('./files/%s.txt' % fileName,'r') as net64bInit:
frames = net64bInit.readlines()
framesTrans = [self.transfer(f) for f in frames]
#写入64比特的帧数据,fileName64b_reverse.txt
with open('./files/%s64b_reverse.txt' % fileName,'w') as net64b:
temp=[net64b.write(f) for f in framesTrans]
mylog.info('正在将64bit的帧文件转换为.npy文件...')
#读入按字节倒序的64比特的帧数据，并转成bytes类型数据，存入.npy文件中
frames = framesTrans
framesNum = len(frames)
tempData = numpy.zeros(framesNum).astype('uint64')
for i,f in enumerate(frames):
tempData[i] = int(f,2)
tempData.dtype = 'uint8'
frameBytes = bytes(tempData)
numpy.save('./files/%s.npy' % fileName,numpy.array(frameBytes))
mylog.info('帧文件转换完成。')
def leave(self, index):
mylog.info("parkinglot.leave()")
self.has_parking()
idx = 0
try:
idx = int(index)
except ValueError:
raise Exception("Invalid Argument: " + str(index))
if (idx > self.n_lots):
raise Exception("Invalid slot number. " +
" Total slot's number :" + str(self.n_lots))
idx = idx - 1
if self.lots[idx]:
self.lots[idx] = None
mylog.info("[RETURN]=True")
return True
else:
raise Exception("No Car parked at slot " + str(index))
def post(self, message, isRetry=False):
mylog.info('Publishing on queue {name}: {data!r}'.format(
name=self.name, data=message))
with self.lock:
if self.connection.is_closed:
mylog.warning("Whoops, connection is closed; reopen.")
self.connection = connection()
self.channel = self.connection.channel()
try:
self.channel.basic_publish(exchange='',
routing_key=self.name,
body=json.dumps(message))
self.maybe_log_message(message)
except Exception as e:
mylog.error(
"Connection failed anyway? Make sure RabbitMQ is running! (is_closed = {})"
.format(self.connection.is_closed))
mylog.error(e.__repr__())
mylog.error(e.args)
mylog.info("Message dropped.")
if not isRetry:
self.post(message, True)
def _remove_citizen(self, tid, token):
with self.lock:
mylog.info("Want to remove citizen {}, token {}".format(
tid, token))
if tid not in self.citizens:
mylog.warning("=> Already deleted (huh?)")
elif self.citizens[tid]['token'] != token:
mylog.info("=> Token mismatch, db has {}".format(
self.citizens[tid]['token']))
else:
mylog.info("=> Yup")
self.twitter.deregister([tid])
del self.citizens[tid]
mylog.info("Remaining citizens: {}".format(
self.citizens.keys()))
def manual_test_incoming():
mylog.info("Reading from Twitter in stand-alone mode.")
twi = RealTwitterInterface()
mylog.info("Now following @HoT *and* Ben's throwaway account")
# If commented out, add 'while True: input()' at the very end:
twi.register_longlived(["4718199753"]) # HouseOfTweetsSB
twi.register_shortlived("774336282101178368", "eeQu0Ae4")
twi.register_shortlived("139407967", "SevimDagdelen")
mylog.info("Kill with Ctrl-C")
def check_config(self, cname):
'''|
| Check test configuration
| Determine the end-of-simulation conditions: len of stim_ and ref_ lists
|________'''
mylog.info(cname + ": " + self.impl2string(self.models))
conf_str = self.dict2string(self.tb_config)
if conf_str != '':
mylog.info("Configuration: " + conf_str)
if self.dut_params != {}:
mylog.info("DUT parameters: " + self.params2string(self.dut_params))
mty_stim_list = True
stim_not_present = True
for key,val in self.tst_data.iteritems():
if "stim" in key:
stim_not_present = False
if val != [] and "file" in val[0]:
num_payloads = sum(1 for line in open(val[0]["file"]))
self.cond_sim_end[key[5:]] = num_payloads
else:
self.cond_sim_end[key[5:]] = len(val)
if self.cond_sim_end[key[5:]] > 0:
mty_stim_list = False
if stim_not_present or mty_stim_list:
mylog.info("No stimuli provided")
no_ref_data = True
for key,val in self.ref_data.iteritems():
if val[0] != [] and "file" in val[0][0]:
num_payloads = sum(1 for line in open(val[0][0]["file"]))
self.cond_sim_end[key] = num_payloads
else:
self.cond_sim_end[key] = len(val[0])
if self.cond_sim_end[key] > 0:
no_ref_data = False
# To avoid endless simulation
if self.tb_config["simulation_time"] == "auto" and (stim_not_present or mty_stim_list) and no_ref_data:
self.tb_config["simulation_time"] = 5
mylog.warn("Simulation time ca not be determined! Selecting a constant simulation time.")
def ischange(reportfile,localreportfile):
"""判断report文件是否有改变"""
if not os.path.exists(localreportfile):
localreportfile = file(localreportfile,'w')
localreportfile.close()
mylog.info('文件创建成功')
return True
else:
if not filecmp.cmp(reportfile,localreportfile):
mylog.info('report对比有改变')
return True
else:
mylog.info('report没改变，不做操作')
return False
def check_process(self,path):
"""查找是否有对应pid"""
path = path.decode('gbk').encode('utf-8')
path_list = path.split(' ')
pid_list = psutil.get_pid_list()
sub_list = []
for pid in pid_list:
cmdline = psutil.Process(pid).cmdline
#print cmdline
if path in cmdline:
mylog.info('程序进程号为 %s'%pid)
return pid
#bug2 执行路径 调用路径有空格 以及部分路径重复都能处理；其他情况未知
elif set(path_list).issubset(cmdline):
for i in xrange(len(cmdline)):
if cmdline[i]==path_list[0]:
sub_list.append(cmdline[i:i+len(path_list)])
if path_list in sub_list:
mylog.info('程序进程号为 %s'%pid)
return pid
mylog.info('未找到程序进程')
return False
)
homs_to_omit = args.homs_to_omit
GENE_NAME = args.gene_name
allow_snps_in_homs = args.allow_snps_in_homs
allow_snps_in_homs_distance = args.allow_snps_in_homs_distance
allow_snps_in_homs_maxNum = args.allow_snps_in_homs_maxNum
rewrite_pair_file = args.rewrite_pair_file
info(
"Inputs:{},{},{},{} | Outputs: {},{},{},{},{},{} ".format(
input_bed,
input_design,
h1_genome_hit_file,
h2_genome_hit_file,
hom_disqualification_log,
mip_disqualification_log,
output_bed,
output_design,
H1_H2_fasta_database,
H1_H2_fasta_query,
)
)
# QC CHECKS
# TODO: check to make sure genome hits file exist
h1_genome_hits, h2_genome_hits = (
pickle.load(open(h1_genome_hit_file, "rb")),
pickle.load(open(h2_genome_hit_file, "rb")),
)
#!/usr/bin/env python3
import mq
import mylog
from twitterConnection import TwitterConnection
from politicianBackend import PoliticianBackend
from citizenQueueAdapter import CitizenQueueAdapter
from birdBackend import BirdBackend
from citizenBirdQueueAdapter import CitizenBirdQueueAdapter
from twitter import RealTwitterInterface
from messages import UpdatesQueueAdapter
birdBack = BirdBackend()
polBack = PoliticianBackend()
follow = polBack.getAllTwitteringPoliticians()
mylog.info("Configured to follow {} accounts.".format(len(follow)))
queue = mq.Batcher(mq.RealQueue("tweets", log_file='all_tweets.json'))
updates = UpdatesQueueAdapter(mq.RealQueue("citizenUserFeedbackQueue"))
twi = TwitterConnection(queue, follow, polBack, birdBack, RealTwitterInterface(), updates)
c = CitizenQueueAdapter(twi)
c.start()
cbq = CitizenBirdQueueAdapter(polBack)
cbq.start()
mylog.info('Backend started')
sequence = design_source.seq[design_source.hom_name == hom_name].values[0]
return sequence
def revcomp(dna, reverse=True):
bases = 'ATGCTACG'
complement_dict = {bases[i]:bases[i+4] for i in range(4)}
if reverse:
dna = reversed(dna)
result = [complement_dict[base] for base in dna]
return ''.join(result)
make_pairs = True
#load the design
info("Loading design file - {}".format(input_design))
design_source = pd.read_table(input_design)
design_source = design_source[design_source.region_index == exon_num]
chromosome = int(design_source.chrom.head(1).values)
## Old version, replace by Jiang at line 159
"""
# Use repeatMasker program
if os.path.isfile(rm_file):
for seq_line in open(rm_file):
if seq_line.startswith(">"):
(chrom, start, stop) = seq_line[1:].rstrip().split("-")
current_index = int(start)
else:
def callback(self, ch, method, properties, body):
body = json.loads(body.decode('utf-8'))
mylog.info("set citizen bird: {}".format(body))
self.politicianBackend.setBird(body["politicianid"], body["birdid"], actor='c')
def callback(self, ch, method, properties, body):
body = json.loads(body.decode('utf-8'))
mylog.info("Add/set citizen user: {}".format(body))
user = body["twittername"]
bird = body["birdid"]
self.twitterConnection.addCitizen(user, bird)
#author：linxs
#version v0.3
# -*- coding:gbk -*-
import pyodbc
import os
import ConfigParser
import mylog
mylog = mylog.initlog()
config = ConfigParser.ConfigParser()
#获取配置文件详细信息
with open("usesql.config", "r+") as cfgfile:
config.readfp(cfgfile)
linkdata = config.get("info", "linkdata")
try:
consql = pyodbc.connect(linkdata)
mylog.info("数据库连接成功")
cursor1 = consql.cursor()
mylog.info("游标建立成功")
except pyodbc.Error, dberror:
mylog.error("数据库连接失败")
mylog.error(dberror)
#此版本values值有两行，会报错hy000
path = os.getcwd()+os.sep+"sql"
sqldir = os.listdir(os.getcwd()+os.sep+"sql")
#取出sql文件最后修改时间，并排序
files = []
#for file in sqldir:
# if file[-4:]=='.sql':
# fileName = os.path.join(path,file)
def consumeTweet(self, tweet):
self.prev_msg_id += 1
mylog.info("(" * 80)
mylog.info("Received tweet #{msg_id}:".format(msg_id=self.prev_msg_id))
mylog.debug(tweet)
# Boring stuff
msg = dict()
msg['content'] = tweet['content']
msg['hashtags'] = tweet['hashtags']
msg['id'] = self.prev_msg_id
msg['image'] = tweet['profile_img']
msg['name'] = tweet['userscreen']
msg['retweet'] = tweet['retweet'] or tweet['content'].startswith('RT ')
msg['time'] = tweet['time']
msg['twitterName'] = tweet['username']
poli = self.pb.getPolitician(tweet['uid'])
citi = self.tw.getCitizen(tweet['uid'])
# Resolve politician/citizen specifics
if poli is not None:
mylog.info("This is definitely a politician.")
msg['poli'] = poli['pid']
birds = self.handle_poli(tweet, msg, poli)
elif citi is not None:
mylog.info("This is definitely a citizen.")
msg['poli'] = None
birds = self.handle_citizen(citi, msg)
else:
mylog.info("Outdated tweet by no-longer citizen {}".format(tweet['uid']))
birds = None
# Make a sound
if birds is None:
mylog.info("=> drop tweet, DONE")
mylog.info(")" * 80)
return
cBird, pBird = birds
msg['sound'] = generate_sound(tweet['content'], tweet['retweet'], cBird, pBird)
# Send it
self.sendingQueue.post(msg)
mylog.info("Done with this tweet, DONE")
mylog.info(")" * 80)
return mType
# Main Program
output_fh = open(output_file, "w")
header = "#Gene\tMutation AA\tMutation cDNA\tChr\tStart\tEnd\tRef\tAlt\tGene Strand\tCOSMIC\tTumor type\tMIP_Mut_Alignment\tGapFillBase_M\tGapFillBase_W\tUpstream\tDownstream\tHom strand\textracted Hom2\textracted Hom1\tHom2 tm\tHom1 tm\tHom2 len\tHom1 len\tH2 GC\tH1 GC\tFwd(50bp+Mutation+50bp)\tRev\tUpstream pos\tDownstream pos\tMIP_Mut_Alignment2\n"
output_fh.write(header)
for line in open(input_file):
parts = line.rstrip().split("\t")
if line.startswith("Gene"):
continue
info("Doing " + "|".join(parts[:3]))
gene, mutation_AA, mutation_cDNA, chrome, start, end, ref, alt, gene_strand, cosmic, tumor_type = parts[
0], parts[1], parts[2], parts[3], int(parts[4]), int(
parts[5]), parts[6], parts[7], parts[8], parts[9], parts[10]
## Only works for SNP
## Situation 1: SNP_on_GF (snp in gap fill)
flank5 = nibFragger(chrome.replace("chr", ""), start - 50, 50)
flank3 = nibFragger(chrome.replace("chr", ""), end + 1, 50)
forwardSeq = flank5.lower() + "[" + ref + "/" + alt + "]" + flank3.lower()
reverseSeq = revcomp(flank3).lower() + "[" + revcomp(ref) + "/" + revcomp(
alt) + "]" + revcomp(flank5).lower()
resultStr = ""
## Take a gene symbol name and output all its exon in bed format
parser = argparse.ArgumentParser(description = "Export gene's exons in bed format from it's canonical model")
parser.add_argument("--gene_name", dest = "gene_name", help = "Gene Symbol", required = True)
parser.add_argument("--output_file", dest = "output_file", help = "Output file name", required = True)
try:
args = parser.parse_args()
except:
print("")
parser.print_help()
sys.exit(1)
gene_name = args.gene_name
output_file = args.output_file
import GenomeInformation as gi
ginfo = gi.GeneInformation()
## check whether gene exists
if ginfo.geneExist(gene_name):
info("Your input is '{}' and output is '{}'".format(gene_name,output_file))
ginfo.makeGeneExonBed(gene_name,output_file)
info("Finished")
else:
error("Gene '"+gene_name+"' not exists\n")
def check_writeback():
mylog.info("politicianBackend: SKIP_WRITEBACK is currently {}".format(_SKIP_WRITEBACK))
