def on_calculateBtn_clicked(self):
"""
Slot documentation goes here.
"""
# TODO: not implemented yet
#raise NotImplementedError
if len(self.ids) == 0:
QMessageBox.warning(self,'warning', u'所选代码为空，请选择代码')
return
startD = self.cmpDateEdit1.date().toPyDate()
endD = self.cmpDateEdit2.date().toPyDate()
if startD == endD:
QMessageBox.warning(self,'warning', u'起止时间相同')
return
#response=json&page=2&pagesize=20&stockid=000001,000002,000003,000004,000005&starttime=2008-09-24&sortname=turnover_ratio
#optname=avg_price,growth_ratio,current_price&opt=-&starttime=2008-03-25&endtime=2008-03-28&page=4&pagesize=20
optname = self.cmpTypeNames[str(self.cmpTypeCombo.currentText().toUtf8()).decode('utf-8')] #self.cmpTypeCombo.currentText().toInt()[0]
opt = self.cmpMethNames[str(self.cmpMethCombo.currentText().toUtf8()).decode('utf-8')]
log(opt)
if opt == 'seperate':
args = {'stockId':','.join(self.ids), 'starttime':startD, 'endtime':endD}
self.calcModel2.setRestApi('listGrowthAmpDis')
else:
args = {'stockId':','.join(self.ids), 'starttime':startD, 'endtime':endD, 'optname':optname, 'opt': opt}
self.calcModel2.setRestApi('List2DiffStockDayInfo')
if len(self.ids) == len(myGlobal.id2name.keys()):
args.pop('stockId')
self.calcModel2.setRestArgs(args)
if not self.tableView_3.inited:
self.tableView_3.myInit(self.calcModel2, 0)
def check(self, fenc='utf8'):
for id in self._input.ids():
#用员工编码查的不到的用姓名查
input_info = self._input[id]
p = input_info[u'姓名']
jx_info = self._jixiao[id]
if not jx_info:
jx_info = self._jixiao[p]
kq_info = self._kaoqin[id]
if not kq_info:
kq_info = self._kaoqin[p]
if not jx_info:
log(WARN, 'id=%s,name=%s have no jixiao' % (id, p))
if not kq_info:
log(WARN, 'id=%s,name=%s have no kaoqin' % (id, p))
jx_check_fields = [u'绩效']
jx_check_fields = []
kq_check_fields = [u'病假小时数', u'事假小时数', u'探亲假小时数', u'年休假小时数', u'夜班天数', u'迟到早退']
if jx_info:
for k in jx_check_fields:
if input_info[k] != jx_info[k]:
print p, input_info[k], jx_info[k]
if kq_info:
for k in kq_check_fields:
if not kq_info[k]:
continue
print p, input_info[k], kq_info[k]
continue
if input_info[k] != kq_info[k]:
print p, input_info[k], kq_info[k]
def login(self):
for get_count in range(10):
out.print_line('Getting uuid', True)
while self.get_QRuuid(): time.sleep(1)
out.print_line('Getting QR Code', True)
if self.get_QR():
break
elif get_count >= 9:
out.print_line('Failed to get QR Code, please restart the program')
sys.exit()
out.print_line('Please scan the QR Code', True)
while self.check_login(): time.sleep(1)
self.web_init()
self.show_mobile_login()
while 1:
voidUserList = self.get_contract()
if not voidUserList: break
out.print_line('These uses need new RemarkNames and are added to a group', True)
chatRoomName = self.create_chatroom(voidUserList, 'RemarkNames')
self.delete_member(chatRoomName, [voidUserList[0]])
self.add_member(chatRoomName, [voidUserList[0]])
while raw_input('Enter "ready" after you rename all of them and DELETE the group: ') != 'ready': pass
out.print_line('Start reload contract list', False)
out.print_line('Login successfully as %s\n'%(
self.storageClass.find_nickname(self.storageClass.userName)), False)
log.log('SIGN IN', True)
thread.start_new_thread(self.maintain_loop, ())
def loadConfig(cfg_fn,defaults):
if not os.path.exists(cfg_fn):
return
fd = open(cfg_fn, 'r')
try:
for line in fd.readlines():
line = line.strip()
if line.startswith('#'):
continue
try:
parts = line.split('=')
key = parts[0]
val = parts[1]
except:
log('Ignoring malformed input line: ', line)
continue
if defaults.has_key(key):
if isinstance(defaults[key],int):
defaults[key] = int(val)
else:
defaults[key] = val
else:
log('Ignoring unknown config variable', key)
finally:
fd.close()
def collectData(self):
"""Collect process list.
"""
self.data.datahash = {} # dict of processes keyed by pid
self.data.proclist = [] # list of processes
self.data.nameHash = {} # dict of processes keyed by process name
procs = win32process.EnumProcesses()
for pid in procs:
try:
han = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION|win32con.PROCESS_VM_READ, 0, pid)
except:
# privileges may prevent querying the process details
han = None
p = proc(pid, han)
if han:
han.Close()
self.data.proclist.append(p)
self.data.datahash[p.pid] = p
self.data.nameHash[p.procname] = p
log.log( "<proc>procList.collectData(): new proc list created", 7 )
def go(codes):
db = operDB.DBOperation()
todayUrl = "http://m.weather.com.cn/data/"
for code in codes:
url = todayUrl + code.strip('\n') + '.html'
log.log("------------------------------------start------------------------------------------")
strLog = "Getting %s datas;" % url
log.log(strLog)
print strLog
myScrap = scrap.weatherScrap(url)
allInfo = myScrap.getWeatherInfo()
if 0 == len(allInfo) :
continue
tablesInfo = publicFun.getFieldById(allInfo)
for i in range(len(tablesInfo)):
db.insertData(tablesInfo[i], publicFun.tables[i])
for i in range(len(tablesInfo)):
db.readTable(publicFun.tables[i])
#deal current weather
currentUrl = "http://www.weather.com.cn/data/sk/"
db.deleteTable('currentweather')
for code in codes:
url = currentUrl + code.strip('\n') + '.html'
myScrap = scrap.weatherScrap(url)
allInfo = myScrap.getWeatherInfo()
if 0 == len(allInfo) :
continue
currentweather = publicFun.getCurrentTableField(allInfo)
db.insertData(currentweather, 'currentweather')
db.readTable('currentweather')
def run(self):
log('I', 'Listening to Redis Channel')
while (True):
shouldEnd = self.listenToChannel(self._pubsub_obj, self._redis_obj)
if (shouldEnd):
break
log('I', 'Ending Listing to Redis Channel')
def regEffect(args):
try:
type = int(args[0])
except:
log.log("est_effect <1/2/3/6/8/7/10/4/5/9/11> <args>")
return
if type == 1:
regEffect1(args[1:])
elif type == 2:
regEffect2(args[1:])
elif type == 3:
regEffect3(args[1:])
elif type == 10:
regEffect10(args[1:])
elif type == 4:
regEffect4(args[1:])
elif type == 5:
regEffect5(args[1:])
elif type == 6:
regEffect6(args[1:])
elif type == 7:
regEffect7(args[1:])
elif type == 8:
regEffect8(args[1:])
elif type == 9:
regEffect9(args[1:])
elif type == 11:
regEffect11(args[1:])
elif type == 12:
regEffect12(args[1:])
else:
log.log("est_effect: unkown type: %s" % type)
def stop_download(download_id):
log.log(__name__, sys._getframe().f_code.co_name, 'download_id %d' % download_id, log.LEVEL_DEBUG)
download_to_stop = ManageDownload.get_download_by_id(download_id)
log.log(__name__, sys._getframe().f_code.co_name, 'download to stop %s' % (download_to_stop.to_string()), log.LEVEL_DEBUG)
ManageDownload.stop_download(download_to_stop)
def load():
if not spe0:
log.log("Unloading spe_effect! Need install spe!!!")
es.unload("wcs/addons/est_effect")
# es.server.insertcmd('sm plugins unload wcs_effects')
cmdlib.registerServerCommand("est_effect", regEffect, "")
# cmdlib.registerServerCommand('est_Effect', regEffect, '')
cmdlib.registerServerCommand("est_effect_01", regEffect1, "")
cmdlib.registerServerCommand("est_effect_02", preRegEffect2, "")
cmdlib.registerServerCommand("est_effect_03", preRegEffect3, "")
cmdlib.registerServerCommand("est_effect_04", regEffect4, "")
cmdlib.registerServerCommand("est_effect_05", preRegEffect3, "")
cmdlib.registerServerCommand("est_effect_06", preRegEffect6, "")
cmdlib.registerServerCommand("est_effect_07", preRegEffect7, "")
cmdlib.registerServerCommand("est_effect_08", preRegEffect8, "")
cmdlib.registerServerCommand("est_effect_09", preRegEffect9, "")
cmdlib.registerServerCommand("est_effect_10", preRegEffect10, "")
cmdlib.registerServerCommand("est_effect_11", preRegEffect11, "")
cmdlib.registerServerCommand("est_effect_12", preRegEffect12, "")
cmdlib.registerServerCommand("est_effect_13", preRegEffect13, "")
cmdlib.registerServerCommand("est_effect_14", preRegEffect14, "")
cmdlib.registerServerCommand("est_effect_15", preRegEffect15, "")
cmdlib.registerServerCommand("est_effect_16", preRegEffect16, "")
cmdlib.registerServerCommand("est_effect_17", preRegEffect17, "")
cmdlib.registerServerCommand("est_effect_18", preRegEffect18, "")
cmdlib.registerServerCommand("est_effect_19", preRegEffect19, "")
cmdlib.registerServerCommand("est_effect_20", preRegEffect20, "")
cmdlib.registerServerCommand("est_effect_21", preRegEffect21, "")
cmdlib.registerServerCommand("est_effect_22", preRegEffect22, "")
cmdlib.registerServerCommand("est_effect_23", preRegEffect23, "")
cmdlib.registerServerCommand("est_effect_24", preRegEffect24, "")
def preRegEffect11(args):
if len(args) != 14 and len(args) != 10:
log.log(
'est_effect_11 <player Filter> <delay> <model> <origin "X Y Z"> <direction "X Y Z"> <R> <G> <B> <A> <Amount>'
)
log.log("est_effect_11: get: " + " ".join(args))
return
if len(args) == 10:
x, y, z = args[3].split(",")
x1, y1, z1 = args[4].split(",")
preRegEffect11([args[0], args[1], args[2], x, y, z, x1, y1, z1, args[5], args[6], args[7], args[8], args[9]])
return
users = str(args[0])
if users == "#a":
users = "#all"
delay = float(args[1])
model = str(args[2])
start = (args[3], args[4], args[5])
dir = (args[6], args[7], args[8])
red = int(args[9])
green = int(args[10])
blue = int(args[11])
alpha = int(args[12])
amount = int(args[13])
# if size < 1:
# log.log('est_effect_10: size > 1. get: %s' % size)
# return
spe_effects.bloodStream(users, delay, start, dir, red, green, blue, alpha, amount)
def run():
args = sys.argv
target = None
if len(args) <=1: # no arguments , help
args.append('help')
if len(args) >1 :
commandName = args[1]
if commandName == 'help':
commandName = 'uhelp'
args = args[2:]
try:
command = __import__( 'commands.' + commandName , globals() , locals() , ['run' , 'options'] )
except:
raise
log.error('Invalid commandName: ' + commandName )
return
argInfo = cli.parseArgv(args , command.options)
if len(argInfo['errors']) > 0:
for error in argInfo['errors']:
log.log(error)
log.error('Invalid command line. Try `ursa help <command>`');
return;
command.run(argInfo['params'],argInfo['options'] );
def collectData(self):
"""Collect disk usage data.
"""
dfre = "^([/0-9a-zA-Z]+)\s*\(([/0-9a-zA-Z])\s*\)\s*:\s*([0-9]+)\s*total allocated Kb\s*([0-9]+)\s*free allocated Kb\s*([0-9]+)\s*used allocated Kb\s*([0-9]+)\s*% allocation used"
rawList = utils.safe_popen('df -ktffs', 'r')
rawList.readline() # skip header
self.data.datahash = {} # dict of filesystems keyed by device
self.data.mounthash = {} # dict of filesystems keyed by mount point
prevline = None
for line in rawList.readlines():
if prevline:
line = prevline + " " + line # join any previous line to current
prevline = None
fields = string.split(line)
if len(fields) == 1: # if 1 field, assume rest on next line
prevline = line
continue
p = df(fields)
self.data.datahash[fields[0]] = p # dict of filesystem devices
self.data.mounthash[fields[5]] = p # dict of mount points
prevline = None
utils.safe_pclose( rawList )
log.log( "<df>dfList.collectData(): collected data for %d filesystems" % (len(self.data.datahash.keys())), 6 )
def processReceivedMessageData(self, msgData):
log.log("Received data: %s\n" % msgData)
try:
container = serializable.deserialize(msgData)
if 'received' in container.keys():
#Process received confirmation:
index = container['received']
self.network.callback.handleMessage(
messages.Confirmation(localID=self.localID, index=index)
)
elif 'message' in container.keys():
index = container['index']
msg = container['message']
#print "Got message: ", str(msg.__class__)
if isinstance(msg, messages.Connect):
if not (self.localID is None):
raise Exception("Received connect message while already connected")
self.localID = msg.ID
self.dice = msg.dice
self.network.checkForDuplicateConnections(self.localID)
else:
#Send confirmation on non-connect messages:
confirmation = {'received': index}
self.send(serializable.serialize(confirmation) + '\n')
#TODO: filter for acceptable message types, IDs etc. before
#sending them to a general-purpose message handler
self.network.callback.handleMessage(msg)
else:
log.log("Received message with invalid format")
except Exception as e:
log.logException()
def main():
for i in range(10):
t = DownloadThread(queue)
t.setDaemon(True)
t.start()
try:
subreddits = sys.argv[1]
force = False
top = False
pg = 1
for arg in sys.argv:
if arg == '--force':
force = True
if arg == '--top':
top = True
if arg.startswith('--pages:'):
pg = int(arg.split(':')[-1])
for subreddit in subreddits.split(','):
app = Client(subreddit,pg, force, top)
app.run()
queue.join()
download.IMAGE_Q.join()
except IndexError: #no arguments provided
log.log(helptext,error=True)
#gui.main()
except KeyboardInterrupt:
log.log('KeyboardInterrupt recieved.',error=True)
sys.exit(1)
def handle_accept(self):
try:
sock, addr = self.accept()
log.log('Incoming connection from %s' % repr(addr))
self.network.makeConnectionFromSocket(sock)
except:
log.logException()
def closeInterface(self, localID):
for i in range(len(self.connections)):
if self.connections[i].localID == localID:
log.log('Closing connection %s' % localID)
self.connections[i].close()
del self.connections[i]
break
def GET(self):
log('logout',session.username)
session.username = None
session.login = 0
session.permission = 0
raise web.seeother('/login/')
def _getuptime(self):
"""Get system statistics from the output of the 'uptime' command."""
uptime_cmd = "/usr/bin/uptime"
(retval, output) = utils.safe_getstatusoutput( uptime_cmd )
if retval != 0:
log.log( "<system>system._getuptime(): error calling '%s'"%(uptime_cmd), 5 )
return None
uptime_re = ".+up (?P<uptime>.+),\s*(?P<users>[0-9]+) users?,\s+ load average:\s+(?P<loadavg1>[0-9.]+),\s*(?P<loadavg5>[0-9.]+),\s*(?P<loadavg15>[0-9.]+)"
inx = re.compile( uptime_re )
sre = inx.search( output )
if sre:
uptime_dict = sre.groupdict()
else:
log.log( "<system>system._getuptime(): could not parse uptime output '%s'"%(output), 5 )
return None
# convert types
uptime_dict['users'] = int(uptime_dict['users'])
uptime_dict['loadavg1'] = float(uptime_dict['loadavg1'])
uptime_dict['loadavg5'] = float(uptime_dict['loadavg5'])
uptime_dict['loadavg15'] = float(uptime_dict['loadavg15'])
return uptime_dict
def makeRouteOutgoing(self, msg):
routeID = self.__makeRouteID(msg.transactionID, msg.isPayerSide)
isOutgoing = msg.isPayerSide
#Try the channels one by one:
for i, c in enumerate(self.channels):
#Reserve funds in channel.
try:
c.reserve(isOutgoing, routeID, msg.startTime, msg.endTime, msg.amount)
except Exception as e:
#TODO: make sure the state of the channel is restored?
log.log("Reserving on channel %d failed: returned exception \"%s\"" % (i, str(e)))
continue
log.log("Reserving on channel %d succeeded" % i)
msg = copy.deepcopy(msg)
msg.ID = self.remoteID
msg.channelIndex = i
return \
[
messages.OutboundMessage(localID=self.localID, message=msg)
]
#None of the channels worked (or there are no channels):
#TODO: haveNoRoute
return []
def settleCommitOutgoing(self, msg):
transactionID = settings.hashAlgorithm(msg.token)
routeID = self.__makeRouteID(transactionID, msg.isPayerSide)
try:
c, ci = self.__findChannelWithRoute(routeID)
except RouteNotInChannelsException:
log.log('No channel found for route; assuming settleCommitOutgoing was already performed, so we skip it.')
return []
ret = self.handleChannelOutput(
ci,
c.settleCommitOutgoing(routeID, msg.token)
)
#TODO: add payload
msg = copy.deepcopy(msg)
msg.ID = self.remoteID
return ret + \
[
messages.OutboundMessage(localID=self.localID, message=msg),
messages.FilterTimeouts(function = lambda message: \
not(
isinstance(message, messages.LinkTimeout_Commit)
and
message.transactionID == transactionID
and
message.isPayerSide == msg.isPayerSide
and
message.ID == self.localID
))
]
def __trap_filter(self, vm):
log.log("brk", "trap filter")
if (vm.cpu.sr.dr6 & (1<<14)) != 0:
return self.__filter_sstep(vm)
tp = None
for i in range(0,4):
if (vm.cpu.sr.dr6 & (1<<i)) != 0:
if (vm.cpu.sr.dr7 & (1<<(i*2+1))) != 0:
conf = (vm.cpu.sr.dr7>>(16+i*4)) & 0xf;
ad = eval("vm.cpu.sr.dr"+str(i))
tp = (conf & 3)+1
ln = ((conf>>2) & 3)+1
break
if tp is None:
ad = vm.cpu.code_location()
ln = 1
for b in self.__list.values():
if b.type == tp and b.addr == ad and b.size == ln:
if b.filter is not None:
return b.filter(vm)
return self.__trap_break(vm)
log.log("brk", "Hardware breakpoint not found !")
return True
def restart(self, cmd):
# cmd is cmd to be executed with: '/etc/init.d/cmd start'.
# cmd should not contain any path information, hence if '/'s are found it
# is not executed.
# Substitute variables in string
cmd = utils.parseVars( cmd, self.varDict )
# Security: if cmd contains any illegal characters, "/#;!$%&*|~`", then we abort.
#if string.find( cmd, '/' ) != -1:
if utils.charpresent( cmd, '/#;!$%&*|~`' ) != 0:
log.log( "<action>action.restart(): Alert, restart() arg contains illegal character and is not being executed, cmd is '%s'" % (cmd), 5 )
return 1001
if len(cmd) == 0:
log.log( "<action>action.restart(): Error, no command given", 5 )
return 1002
# Build command
cmd = '/etc/init.d/'+cmd+' start'
# Call system() to execute the command
log.log( "<action>action.restart(): calling os.system() with cmd '%s'" % (cmd), 6 )
# TODO: possibly not thread-safe to simply call os.system() without
# locking; this might have to be part of the thread-safe group of
# system commands in utils.py
retval = os.system( cmd )
# Alert if return value != 0
if retval != 0:
log.log( "<action>action.restart(): Alert, return value for cmd '%s' is %d" % (cmd,retval), 5 )
log.log( "<action>action.restart(): cmd '%s', return value %d" % (cmd,retval), 6 )
return retval
def dump_registers(self):
"""
Log the register values
"""
for key, val in self.register_legend.iteritems():
log.log("{:12} {:>2}: {:>10} {:>10}".format(
key, val[0], self.registers[val[0]], "0x{:X}".format(self.registers[val[0]])))
def learn(self, input_, cat, gcr=None):
"""
input_ of len 1-256 (limit to 256 to properly emulate a CM1K).
cat 0-32767. 1-32767: vector category. 0: counterexample (shrink aifs, but don't commit RTL neuron).
gcr 0-127. 1-127: context. 0: use all neurons (disregard context). None: don't overwrite current context.
"""
log.trace("CM1KEmulator.learn()")
# log.log("Input: {},{}: {}".format(gcr, cat, input_))
# self.dump_registers()
self.broadcast(input_, gcr)
self.write_cat(cat)
min_firing_dist = self.firing_neurons[-1].dist if self.firing_neurons else self.read_maxif()
minif = self.read_minif()
# Shrink any misfiring neurons to the shortest distance of any firing neuron
for neuron in self.firing_neurons:
neuron.shrink_if_necessary(cat, min_firing_dist, minif)
# Determine if any neurons correctly fired
any_correctly_firing_neuron = False
for neuron in self.firing_neurons:
if neuron.cat == cat:
any_correctly_firing_neuron = True
break
log.log("any_correctly_firing_neuron: {}".format(any_correctly_firing_neuron))
# for neuron in self.neurons:
# log.log("(A) Neuron: {}".format(neuron.dump()))
# If there are no correctly firing neurons, consider recruiting a new neuron to hold the pattern as a new prototype
if not any_correctly_firing_neuron:
rtl_neuron_idx = None
if not self.unlimited_neurons:
# Find the RTL neuron
# This should be stored in the registers, but it's safer to just find it explicitly
for i, neuron in enumerate(self.neurons):
# log.log("RTL search, neuron: {} {}".format(neuron.id_, neuron.state))
if neuron.state == nrn.NeuronState.rtl:
rtl_neuron_idx = i
break
else:
# If the number of neurons is unlimited, then the RTL neuron is simply at the end of the list
rtl_neuron_idx = len(self.neurons) - 1
# Assign the RTL neuron the input as a new pattern
if rtl_neuron_idx is not None:
log.log("rtl_neuron: idx{}, id{}".format(rtl_neuron_idx, self.neurons[rtl_neuron_idx].id_ if self.neurons[rtl_neuron_idx] else None))
new_aif = min(max(min_firing_dist, minif), self.read_maxif())
log.log("new_aif: {}".format(new_aif))
self.neurons[rtl_neuron_idx].commit(self.read_gcr_context(), cat, new_aif, input_)
log.log("Committed: id{} st{}".format(self.neurons[rtl_neuron_idx].id_, self.neurons[rtl_neuron_idx].state))
if self.unlimited_neurons:
self.neurons.append(nrn.Neuron(len(self.neurons), self))
if self.neurons[rtl_neuron_idx].id_ < len(self.neurons) - 1:
self.neurons[self.neurons[rtl_neuron_idx + 1].id_].state = nrn.NeuronState.rtl
self.write_total_ncount_non_ui(self.read_ncount() + 1)
else:
log.log("RTL neuron is None")
def collectData(self):
"""
Collect network interface data.
"""
self.data.datahash = {} # hash of same objects keyed on interface name
self.data.numinterfaces = 0
# get the interface statistics
fp = open('/proc/net/dev', 'r')
# skip header lines
fp.readline()
fp.readline()
for line in fp.readlines():
(name,data) = string.split( line, ':' ) # split interface name from data
f = string.split(data)
t = interface(f) # new interface instance
if t == None:
log.log( "<netstat>iftable: error parsing interface data for line '%s'"%(line), 5 )
continue # could not parse interface data
t.name = string.strip(name)
self.data.datahash[t.name] = t
self.data.numinterfaces = self.data.numinterfaces + 1 # count number of interfaces
fp.close()
log.log( "<netstat>IntTable.collectData(): Collected data for %d interfaces" %(self.data.numinterfaces), 6 )
def collectData(self):
self.data.datalist = [] # list of TCP objects
self.data.datahash = {} # hash of same objects keyed on '<ip>:<port>'
self.data.numconnections = 0
# get list of current TCP connections
rawList = utils.safe_popen('netstat -an -t', 'r')
# skip header line
rawList.readline()
for line in rawList.readlines():
f = string.split(line)
if len(f) != 6:
continue # should be 7 fields per line
t = tcp(f) # new TCP instance
self.data.datalist.append(t)
self.data.datahash['%s:%s' % (t.local_addr_ip,t.local_addr_port)] = t
self.data.numconnections = self.data.numconnections + 1 # count number of TCP connections
utils.safe_pclose( rawList )
log.log( "<netstat>TCPtable.collectData(): Collected %d TCP connections" %(self.data.numconnections), 6 )
def __init__(self, host, port) :
self.servHost = host
self.servPort = port
self.sock = socket.socket()
self.sock.connect((host, port))
log('C','TCPClient Initialized, Server[%s:%s] connected'%(host,port))
def collectData(self):
self.data.datalist = [] # list of UDP objects
self.data.datahash = {} # hash of same objects keyed on '<ip>:<port>'
self.data.numconnections = 0
# get the UDP stats
rawList = utils.safe_popen('netstat -anA inet -u', 'r')
# skip header lines (2)
rawList.readline()
rawList.readline()
for line in rawList.readlines():
f = string.split(line)
if len(f) < 5 or len(f) > 6:
continue # should be 5 or 6 fields per line
t = udp(f) # new udp instance
self.data.datalist.append(t)
self.data.datahash['%s:%s' % (t.local_addr_ip,t.local_addr_port)] = t
self.data.numconnections = self.data.numconnections + 1 # count number of UDP connections
utils.safe_pclose( rawList )
log.log( "<netstat>UDPtable.collectData(): Collected %d UDP connections" %(self.data.numconnections), 6 )
def setUp(self):
unittest.TestCase.setUp(self)
desired_caps = {}
desired_caps['platformName'] = 'Android'
desired_caps['platformVersion'] = '6.0.1'
desired_caps['deviceName'] = 'a155b926'
desired_caps['appPackage'] = 'com.wuba.moneybox'
desired_caps['appActivity'] = 'com.wuba.moneybox.ui.SplashActivity'
#连接设备并打开应用
self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps)
try:
#有宣传图的滑动宣传图
time.sleep(3)
self.driver.swipe(1030, 1568, 62, 1568, 1000)
log.log(u"滑动宣传图成功！")
time.sleep(2)
self.driver.tap([(538,1655)])
#点击更新提示，取消
self.driver.find_element_by_id("com.wuba.moneybox:id/positiveButton").click()
except:
log.log(u"没找到宣传图！")
time.sleep(2)
def msg_haveNoRoute(self, msg):
log.log('Processing HaveNoRoute message')
try:
if msg.isPayerSide:
tx = self.findTransaction(
transactionID=msg.transactionID, payeeID=msg.ID, isPayerSide=True)
else:
tx = self.findTransaction(
transactionID=msg.transactionID, payerID=msg.ID, isPayerSide=False)
except TransactionNotFound:
log.log(' HaveNoRoute failed: transaction %s does not (or no longer) exist (ignored)' % \
msg.transactionID.encode('hex'))
return []
payer = self.__getLinkObject(tx.payerID)
payee = self.__getLinkObject(tx.payeeID)
ret = []
if tx.isPayerSide:
ret += payee.haveNoRouteIncoming(msg)
else:
ret += payer.haveNoRouteIncoming(msg)
#Clean up old route and lock time-out:
oldPayerID = tx.payerID #keep reference to innermost object
ret.append(messages.FilterTimeouts(function = lambda message: not (
isinstance(message, (messages.NodeStateTimeout_Route, messages.NodeStateTimeout_Lock))
and
message.transactionID == msg.transactionID
and
message.isPayerSide == msg.isPayerSide
and
message.payerID == oldPayerID
)))
#Try to find another route
nextRoute = tx.tryNextRoute()
if nextRoute is None:
log.log(' No remaining route found')
if tx.isPayerSide:
ret += payer.haveNoRouteOutgoing(msg.transactionID, isPayerSide=True)
else:
ret += payee.haveNoRouteOutgoing(msg.transactionID, isPayerSide=False)
#Clean up cancelled transaction:
self.transactions.remove(tx)
return ret
log.log(' Forwarding MakeRoute to the next route')
ret += self.__getLinkObject(nextRoute).makeRouteOutgoing(
messages.MakeRoute(
amount = tx.amount,
transactionID = msg.transactionID,
startTime = tx.startTime,
endTime = tx.endTime,
meetingPointID = tx.meetingPointID,
ID = nextRoute,
isPayerSide = tx.isPayerSide
))
#route time-out:
#TODO: configurable time-out value?
ret.append(messages.TimeoutMessage(timestamp=time.time()+5.0, message=\
messages.NodeStateTimeout_Route(
transactionID=msg.transactionID, isPayerSide=msg.isPayerSide,
payerID=tx.payerID
)))
return ret
def __init__(self, selenium_driver, base_url, page_title):
self.driver = selenium_driver
self.url = base_url
self.title = page_title
self.mylog = log.log()
# >>> moment(curent_time).fromNow(refresh=True)
# #显示中文格式的时间
# >>> moment.lang('ch')"
moment = Moment()
#flask_sqlalchemy模块中提供的SQLAlchemy对象,用来管理数据库
db = SQLAlchemy()
#flask_login模块中的LoginManager对象,用来管理用户登录
login_manager = LoginManager()
#会话保护设置称strong
login_manager.session_protection = None
#设置视图函数
login_manager.login_view = 'auth.login'
#日志管理对象
log = log()
@login_manager.user_loader
def load_user(id):
return User.query.get(int(id))
def create_app(config_name):
"""
创建app的接口,接受配置名称,创建flask程序对象.
Example:
app=create_app('test')
"""
app = Flask(__name__)
app.config.from_object(config[config_name])
conn = sqlite3.connect('static/ledger.db')
else:
conn = sqlite3.connect('static/hyper.db')
conn.text_factory = str
global c
c = conn.cursor()
if full_ledger == 1:
conn2 = sqlite3.connect('static/ledger.db')
else:
conn2 = sqlite3.connect('static/hyper.db')
conn2.text_factory = str
global c2
c2 = conn.cursor()
app_log = log.log("gui.log", debug_level)
essentials.keys_check(app_log)
essentials.db_check(app_log)
mempool = sqlite3.connect('mempool.db', timeout=1)
mempool.text_factory = str
m = mempool.cursor()
# for local evaluation
root = Tk()
root.wm_title("Bismuth")
def help():
def _exitConnection(self, reason):
msg = '[SRV]: Connection refused: {0}\n'.format(reason)
self.write_data(msg)
log(msg)
self.handle_close()
elif mode == 'rand':
if len(old_para_vals) == 0:
random.shuffle(para_vals)
old_para_vals.append(para_vals.pop(0))
else: # add new ways above !
forcelog('get_next_para_val mode is unknown!!')
return None
return old_para_vals[-1]
##MAIN...
if __name__ == "__main__":
if LOAD_CS_CFG:
log('LOAD_CS_CFG mode OFF !!!')
if NON_CS_DEBUG:
log('NON_CS_DEBUG mode ON !!!')
# digitwin.DigiTwin.setRealTimeMode()
# Initialize vectors
sims = []
sim_initiated = []
sim_semaphores = []
para_name = []
para_min = []
para_val = []
para_max = []
para_step = []
runs = []
init_runs = []
def find_version(version):
log.log("Find version %s" % version)
return getattr(versions, version, find_downloaded_version(version))
def token_seachange(path, argument, ip):
passed = False
uri = path # Just for warnings in the exception handler
# Rebuild query string in correct order...
# NOTE: Use more local variables for debugging purposes.
try:
# Check for address
if config.seachange['ip_restrict'] and argument['a'] != ip:
return False
# Check for expired date - SeaChange uses UTC
if config.seachange['expire_date_restrict']:
if config.token_types['seachange'] == "SHA1":
expire_date = calendar.timegm(argument['e'])
else:
end = str(argument['e'])
year = end[:4]
month = end[4:2]
day = end[6:2]
hour = end[8:2]
minute = end[10:2]
second = end[12:]
expire_date = datetime.datetime(year, month, day, hour, minute,
second)
if datetime.datetime.utcnow() > expire_date:
return False
uri = str("%s?i=%s&k=%s&e=%s&a=%s" %
(path, argument['i'], argument['k'], argument['e'],
argument['a'])).lower()
#byte_string = uri.encode('utf-8')
#byte_string = bytearray(uri, 'utf-8')
key = config.keys_seachange[argument['k']]
if config.token_types['seachange'] == "MD5":
token = hmac(key.decode('hex'), uri, md5)
else:
token = hmac(key.decode('hex'), uri, sha1)
hex_value = token.hexdigest()
if hex_value == argument['h']:
passed = True
if config.logging:
if passed:
return_code = config.http_response_codes["found"]
msg = "%s = %s" % (hex_value, argument['h'])
else:
return_code = config.http_response_codes["forbidden"]
msg = "%s != %s" % (hex_value, argument['h'])
log("%s - - \"TOKEN(SC) %s\" %s %s \"%s\" \"%s\" \"-\"" %
(ip, uri, return_code, 0, msg, argument))
except BaseException, e:
log("%s - - \"GET %s\" %s %s \"%s\" \"%s\" \"-\"" %
(ip, uri, config.http_response_codes["forbidden"], 0, e, argument),
level="WARNING",
log_type="ERROR")
import time, config, pid, irc, db, log
pid = pid.pid()
irc = irc.irc()
db = db.db()
log = log.log()
db.get_raffle_status()
if pid.oktorun:
try:
while True:
time.sleep(1)
# Output
out = db.get_next_output(time.time())
if out:
sendStr = 'PRIVMSG '+ out[1] +' :'+ out[2] +'\n'
print(sendStr)
irc.sendmsg(sendStr)
log.logmsg(sendStr, False)
db.delete_output(out[0])
try:
msg = irc.getmsg()
print(msg)
# Reconnect if disconnected
if len(msg) == 0:
irc.connect()
# Prevent Timeout
def msg_makeRoute(self, msg):
log.log('Processing MakeRoute message')
sourceLink = self.__getLinkObject(msg.ID)
ret = sourceLink.makeRouteIncoming(msg)
payerID, payeeID = \
{
True: (msg.ID, None),
False: (None, msg.ID)
}[msg.isPayerSide]
#Possible routes we can take
if msg.routingContext is None:
#Order is important: try meeting points first
possibleLinks = self.meetingPoints.keys() + self.links.keys()
else:
possibleLinks = [msg.routingContext]
def tryRemove(ID):
try:
possibleLinks.remove(ID)
except ValueError:
pass #it's OK if the source link wasn't present already
#Remove the source link:
tryRemove(msg.ID)
#Remove source link and possible routes of earlier instances of
#this route:
#for these, the route should be made by the earlier instance.
#Allowing them to be selected by later instances would allow
#infinite routing loops.
#Note: generally, this will remove ALL routes, if earlier instances of
#the same route exist. The only situation where this is not the case
#is when an earlier instance was restricted in its routing choices,
#and, theoretically, when a new route was created in-between.
earlierTransactions = self.findMultipleTransactions(
transactionID=msg.transactionID, isPayerSide=msg.isPayerSide)
for earlierTx in earlierTransactions:
earlierSourceLinkID = earlierTx.payerID if msg.isPayerSide else earlierTx.payeeID
tryRemove(earlierSourceLinkID)
for ID in earlierTx.initialLinkIDs:
tryRemove(ID)
#Increment end time on the payee side:
#On the payer side, this will be done in haveRoute.
if not msg.isPayerSide:
#TODO: check sanity (and data type) of startTime, endTime
msg.endTime += self.settings.timeoutIncrement
#Create new transaction
newTx = transaction.Transaction(
state=transaction.Transaction.states.makingRoute,
isPayerSide=msg.isPayerSide,
payeeID=payeeID,
payerID=payerID,
initialLinkIDs=possibleLinks[:],
remainingLinkIDs=possibleLinks[:],
meetingPointID=msg.meetingPointID,
amount=msg.amount,
transactionID=msg.transactionID,
startTime=msg.startTime,
endTime=msg.endTime
)
self.transactions.append(newTx)
nextRoute = newTx.tryNextRoute()
if nextRoute is None:
log.log(' No route found')
#Delete the tx we just created:
self.transactions.remove(newTx)
#Send back haveNoRoute:
ret += sourceLink.haveNoRouteOutgoing(
msg.transactionID, msg.isPayerSide)
return ret
log.log(' Forwarding MakeRoute to the first route')
ret += self.__getLinkObject(nextRoute).makeRouteOutgoing(msg)
#route time-out:
#TODO: configurable time-out value?
ret.append(messages.TimeoutMessage(timestamp=time.time()+5.0, message=\
messages.NodeStateTimeout_Route(
transactionID=msg.transactionID, isPayerSide=msg.isPayerSide,
payerID=newTx.payerID
)))
return ret
def TestRunResult(run_spec, job_spec, job_spec_path, run_item, input_data, run_result, result_data, command_options, command_args):
"""Test the run_result for the run_item. Abort, report and exit if we fail any of the tests."""
log('Test Run Result: %s' % run_result)
# List of dicts, with test result information (critical, warning, success)
test_results = []
for test_case in run_item['tests']:
# Skip test case if this when doesnt match
if 'finished' in run_result and test_case['when'] != 'finished':
continue
elif 'finished' not in run_result and test_case['when'] != 'during':
continue
# Ensure we have the test case field key we want to test, or fail
if test_case['key'] not in run_result:
Error('Missing test case key in run result: %s: %s' % (test_case['key'], run_result), command_options)
# Get the value we are to operation on
value = run_result[test_case['key']]
# Create our test_result dict, and just append it to our list of test results now
test_result = {}
test_results.append(test_result)
# Test with specified function
if test_case['function'] in ['==', 'equals']:
# If pass
if value == test_case['value']:
test_result['success'] = True
# If fail
else:
test_result['success'] = False
# Not Equals
elif test_case['function'] in ['!=', 'not equals']:
# If pass
if value != test_case['value']:
test_result['success'] = True
# If fail
else:
test_result['success'] = False
# RegEx
elif test_case['function'] in ['regex']:
regex_result = re.findall(str(test_case['value']), str(value))
# If pass
if regex_result:
test_result['success'] = True
# If fail
else:
test_result['success'] = False
# ---- Test Cases are Finished ----
# If we had a failure
if not test_result['success']:
if test_case.get('log failure', None):
log_message = test_case['log failure'] % run_result
test_result['log'] = log_message
log('Result Test Failure: %s' % log_message)
if test_case['critical']:
test_result['critical'] = True
break
if test_case['warning']:
test_result['warning'] = True
# Else, we had a success
else:
if test_case.get('log success', None):
log_message = test_case['log success'] % run_result
test_result['log'] = log_message
log('Result Test Failure: %s' % log_message)
return test_results
"[skipping] %s can not be published because it has already bean published",
status)
continue
log(
"[publishing] %s : %s",
strftime("%Y-%m-%d %H:%M:%S", publishtime)
if publishtime is not None else 'None',
status,
)
if pub2all(status, entry):
if publishtime is not None and maxtime < publishtime:
maxtime = publishtime
lasttimes['links_' + url].append(entry['link'])
sleep(10)
lasttimes[url] = maxtime if maxtime != lasttime else gmtime()
lasttimes['links_' + url] = lasttimes['links_' + url][-100:]
save_rss_lasttimes(lasttimes)
if __name__ == '__main__':
log("start...")
twitter2all()
feeds2all()
log("finish...")
def read_url(app_log, url):
url_split = url.split("/")
app_log.warning(url_split)
reconstruct = "bis://{}/{}/{}/{}/".format(url_split[2], url_split[3],
url_split[4], url_split[5],
url_split[6])
openfield_b85_decode = base64.b85decode(url_split[5]).decode("utf-8")
if checksum(reconstruct) == url_split[6]:
url_deconstructed = url_split[2], url_split[3], url_split[
4], openfield_b85_decode
app_log.warning("Checksum match")
return url_deconstructed
else:
app_log.warning("Checksum mismatch", checksum(reconstruct),
url_split[6])
return
if __name__ == "__main__":
#test
import log
app_log = log.log("node.log", "WARNING", "yes")
print("create_url", create_url(app_log, "pay", "recipient", "10", "dd611"))
print(
"read_url",
read_url(app_log,
"bis://pay/recipient/10/WMnomF#/`W4$&s&qu}j&z@EM@>rV"))
import time
import multiprocessing
from Mysql_connect import MYSQL
from get_trade_info import InfoFromAPI
from log import log
TRADE_SUIT_LIST = ['insur_eth', 'insur_btc', 'insur_usdt']
COLLECT_INTERVAL_TIME = 10
logger = log()
def main():
logger.info("START!!")
for trade_suit in TRADE_SUIT_LIST:
db_con = MYSQL(trade_suit)
if not db_con.is_table_exist():
db_con.create_table()
db_con.db_shut_off()
pool = multiprocessing.Pool(processes=len(TRADE_SUIT_LIST))
for trade_suit in TRADE_SUIT_LIST:
pool.apply_async(func=collect_and_store, args=(trade_suit, ))
pool.close()
pool.join()
def change_timestamp(date_ms):
time_str = str(date_ms)
time_date = int(time_str[:10])
time_ms = time_str[-3:]
time_date = time.strftime("%Y%m%d-%H%M%S", time.localtime(time_date))
print(
'We are retraining the model (because of low precision). This may take a while. Please wait!'
)
connection.connection.ping() # Ping MySQL to maintain the connection.
tf.reset_default_graph()
# Save the machine-learning models to the database (MLModel), if the test recalls are greater than the target recalls. If not, then email the administrator.
ml_models = []
for model in models_dict:
if (model.get('test_recall') < model.get('target_recall')):
event = 'existential_risk_ml.py'
note = 'Warning! When testing the performance of the newly trained machine-learning model for {topic}, the recall ({test_recall}) was lower than the target recall ({target_recall}). Therefore, the performance of this model was not saved in MLModel and its predictions were not saved in MLPrediction. If the target recall was 0.50, then you might need to set a value that is greater than 0.33 in thresholds = np.arange(0.0, 0.33, 0.0001).'.format(
topic=search_topic,
test_recall=model.get('test_recall'),
target_recall=model.get('target_recall'))
log(event=event, note=note)
#TODO: Email the administrator.
exit()
ml_models.append(
MLModel(topic=search_topic,
threshold=model.get('threshold'),
accuracy=model.get('accuracy'),
precision=model.get('precision'),
target_recall=model.get('target_recall'),
test_recall=model.get('test_recall')))
# Predict the relevance of publications that have not yet been assessed by humans.
unassessed_publications = Publication.objects.distinct().filter(
search_topics=search_topic).exclude(
assessment__in=Assessment.objects.all())
df = pd.DataFrame(
try:
job_spec = yaml.load(open(job_spec_path))
except Exception, e:
Error('Failed to load job spec: %s: %s' % (job_spec_path, e), command_options)
# Fail if the platform is not in this job spec
if command_options['platform'] not in job_spec['run']:
Error('This platform is not supported by this job, no run commands available: %s' % command_options['platform'], command_options)
# Initiate run procedures
if not input_data:
log('Retrieving input data manually')
input_data = RetrieveInputData(run_spec, job_spec, job_spec_path, command_options, command_args)
log('Input Data: %s' % input_data)
# Run it...
log('Running job: %s' % job_spec['data']['name'])
# Get the run items for our current platform
result_data = {'started':time.time(), 'run_results':[], 'success':None}
platform_run_items = job_spec['run'][command_options['platform']]
# Run each platform run item
for run_item in platform_run_items:
run_result = RunItem(run_spec, job_spec, job_spec_path, run_item, input_data, command_options, command_args)
def load_log(logfile):
log.default = log.log(streams=[logfile])
def start(self):
# Login
self.login()
# Start farming codes
while(self.logged_in):
# Get a UNIDAYS code
self.get_code()
# Wait for cooldown to end
time.sleep(3900)
if(__name__ == "__main__"):
# Ignore insecure request warnings
requests.packages.urllib3.disable_warnings()
# Get username and password
try:
file = open("settings.txt", "r")
settings = file.readlines()
email = settings[0].strip()
password = settings[1].strip()
ready = True
except Exception as e:
log('e', "Settings file corrupted! Exiting...")
ready = False
# Start farming codes
if(ready):
NikeCAUnidaysFarmer(email, password).start()
parameter = parameters()
parameter.initialize()
if parameter.correctParameters == False:
print('Please update your input files and restart ScMiles.')
sys.exit()
jobs = run(parameter)
samples = sampling(parameter, jobs)
lastLog = ""
# initialize with reading anchor info and identifying milestones
if parameter.restart == False:
parameter.MS_list = milestones(parameter).initialize(status=status)
else:
seek, sample, lastLog = read_log(parameter, status)
if seek == False:
log('ScMiles restarted. Continuing seek step.')
parameter.MS_list = milestones(parameter).initialize(status=status)
elif sample == False:
log('ScMiles restarted. Continuing sampling step.')
parameter.MS_list = milestones(parameter).read_milestone_folder()
else:
#Check for new sampling
log('ScMiles restarted. Continuing free trajectories step.')
parameter.MS_list = milestones(parameter).read_milestone_folder()
if "Preparing for more free trajectories" in lastLog:
parameter.restart = False
lastLog = ""
while True:
free_trajs = traj(parameter, jobs)
#!/usr/bin/python
# Script: logview.py
# Purpose: plots of LAMMPS log-file thermodynamic data
# Syntax: logview.py gnu/matlab files ...
# gnu/matlab = style of plots to create
# files = one or more log files
# Example: logview.py gnu log.*
# Author: Steve Plimpton (Sandia)
# enable script to run from Python directly w/out Pizza.py
import sys
from log import log
from plotview import plotview
from gnu import gnu
from matlab import matlab
if "argv" not in globals(): argv = sys.argv
# main script
if len(argv) < 3:
raise Exception("Syntax: logview.py gnu/matlab files ...")
style = argv[1]
files = ' '.join(argv[2:])
lg = log(files)
exec("plot = %s()" % style)
p = plotview(lg, plot)
def data_parse(cblock_str):
'''Compute the FHR MHR TOCO mother_mv
from C block'''
mm = '10024d4d1003' # MM block
event = 0
FHR = []
MHR = []
TOCO = 0
mother_mv = []
data_one_sec = []
init_An24.run_check(cblock_str)
if (len(cblock_str) == CBLOCK_STR_LEN):
FHR_split = [10, 14, 18, 22, 26]
MHR_split = [42, 46, 50, 54, 58]
TOCO_split = [58, 60, 62, 64, 66]
FHR_section = [
cblock_str[FHR_split[0]:FHR_split[1]],
cblock_str[FHR_split[1]:FHR_split[2]],
cblock_str[FHR_split[2]:FHR_split[3]],
cblock_str[FHR_split[3]:FHR_split[4]]
]
MHR_section = [
cblock_str[MHR_split[0]:MHR_split[1]],
cblock_str[MHR_split[1]:MHR_split[2]],
cblock_str[MHR_split[2]:MHR_split[3]],
cblock_str[MHR_split[3]:MHR_split[4]]
]
TOCO_section = [
cblock_str[TOCO_split[0]:TOCO_split[1]],
cblock_str[TOCO_split[1]:TOCO_split[2]],
cblock_str[TOCO_split[2]:TOCO_split[3]],
cblock_str[TOCO_split[3]:TOCO_split[4]]
]
FHR_section = [int_16(f) for f in FHR_section]
MHR_section = [int_16(m) for m in MHR_section]
TOCO_section = [int_16(t) for t in TOCO_section]
''' :
for fh_s in xrange(0,4):
FHR[fh_s] = round(int(FHR_section[fh_s]&
0x7FF) *HEART_RATE_RESOLUTION, 2)
for mh_s in xrange(0,4):
MHR[mh_s] = round(int(MHR[mh_s]&
0x7ff) * HEART_RATE_RESOLUTION, 2)
mother_mv[mh_s] = int((MHR[mh_s]&0x1800) >> 11)
'''
for fh_s in FHR_section:
fh = round(int(fh_s & 0x7FF) * HEART_RATE_RESOLUTION, 2)
FHR.append(fh)
for mh_s in MHR_section:
mh = round(int(mh_s & 0x7ff) * HEART_RATE_RESOLUTION, 2)
m_mv = int((mh_s & 0x1800) >> 11)
mother_mv.append(m_mv)
MHR.append(mh)
for to_s in TOCO_section:
TOCO = int(int(to_s) * TOTO_RESOLUTION)
'''
print 'FHR:', FHR
print 'MHR:', MHR
print 'TOCO:', TOCO
print 'mother_mv:', mother_mv
'''
elif len(cblock_str) == NBLOCK_STR_LEN:
FHR = [0, 0, 0, 0]
MHR = [0, 0, 0, 0]
mother_mv = [0, 0, 0, 0]
'''Compute the SNR from N block'''
if cblock_str[14:16] == NBLOCK_SNR_TAG:
global SNR
fetal_signal = int(cblock_str[16:24], 16)
noise = int_16(cblock_str[24:32]) #int(cblock_str[24:32], 16)
SNR_func = lambda x, y: round(x / float(y), 4)
SNR = SNR_func(fetal_signal, noise)
#print 'SNR:', SNR
else:
pass
elif cblock_str == mm:
FHR = [0, 0, 0, 0]
MHR = [0, 0, 0, 0]
mother_mv = [0, 0, 0, 0]
event = 1
log('-!-!-!-!-event-!-!-!-!-', event)
else:
FHR = [0, 0, 0, 0]
MHR = [0, 0, 0, 0]
mother_mv = [0, 0, 0, 0]
data_one_sec.append([FHR[0], MHR[0], TOCO, mother_mv[0], SNR, event])
data_one_sec.append([FHR[1], MHR[1], TOCO, mother_mv[1], SNR, event])
data_one_sec.append([FHR[2], MHR[2], TOCO, mother_mv[2], SNR, event])
data_one_sec.append([FHR[3], MHR[3], TOCO, mother_mv[3], SNR, event])
log('data_one_sec:', data_one_sec)
#return FHR, MHR, TOCO, mother_mv, SNR, event
return data_one_sec
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
import re
import json
import os
import sys
import argparse
import collections
import csv
import traceback
from log import log
logger = log(os.path.basename(sys.argv[0]))
LEMMAS_QS_JSON_FILE = u'lemmas/lemmas_qs.json'
LEMMAS_QS_EXTRA_JSON_FILE = u'lemmas/lemmas_qs_extra.json'
REVERSE_LEMMAS_JSON_FILE = u'lemmas/rev_lemmas.json'
class AllText(object):
def __init__(self, file_path):
"""
:param file_path:
生成以下私有变量：
__file_path string, 输入的文件名
__all_text string. 全文
__words list, 未去重的所有单词
__words_cnt dict, 根据原始文本统计的词频，未作任何筛除{word: frequency, ...}
def monitor(link, keywords):
'''
the URL(string) is scanned and alerts
are sent via Discord when a new product containing a keyword(list) is detected.
'''
log('i', "Checking site |" + link + "|...")
# Parse the site from the link
HTTPS = link.find("https://")
HTTP = link.find("http://")
if (HTTPS == 0):
site = link[8:]
end = site.find("/")
if (end != -1):
site = site[:end]
site = "https://" + site
else:
site = link[7:]
end = site.find("/")
if (end != -1):
site = site[:end]
site = "http://" + site
# Get all the links on the "New Arrivals" page
try:
r = requests.get(link, timeout=5, verify=False)
except:
log('e', "Connection to URL <" + link + "> failed. Retrying...")
time.sleep(5)
try:
r = requests.get(link, timeout=8, verify=False)
except:
log('e', "Connection to URL| " + link + " |failed...")
return
page = soup(r.text, "html.parser")
raw_links = page.findAll("a")
hrefs = []
for raw_link in raw_links:
try:
hrefs.append(raw_link["href"])
except:
pass
# Check for links matching keywords
for href in hrefs:
found = False
for keyword in keywords:
if (keyword.upper() in href.upper()):
found = True
if ("http" in href):
product_page = href
else:
product_page = site + href
product = Product("N/A", product_page, True, keyword)
alert = sendToDataBase(product)
if (alert):
send_embed(product)
from tubotModules import createHTML, updateFeedFile, storeChannelData
# TODO: add css file
# TODO: don't use settings.settings - rename it to getSettings or something
# TODO: change to different CSS file in settings
# TODO: addChannel() -> verify channel existence before creating files
# TODO: split tubotModules.py into modules
# TODO: change ChannelTimings in Manager
# TODO: change TubotSettings in Manager
# TODO: exceptions
# TODO: add multiple pages
# TODO: lock main if manager is active
# TODO: maybe rename .feed to .channel
# Todo: replace smelly Code
log('tubot init')
while True:
settings = getSettings('load', None, None)
debug = settings['debug']
timestamp = datetime.now()
# get updates from the web
if debug is True or timestamp.minute == 0:
for channelFile in os.listdir('channels/'):
channelData = getChannelData(channelFile)
if evaluatePeriodicity(channelData, timestamp, debug) is True:
updateChannel(channelFile)
time.sleep(settings['delay'])
elif evaluatePeriodicity(channelData, timestamp, debug) is None:
log('evaluatePeriodicity() returns None', channelData,
timestamp, debug)
def push(self,
board: str,
push_type: int,
push_content: str,
post_aid: str = None,
post_index: int = 0) -> None:
self._one_thread()
if not self._login_status:
raise exceptions.Requirelogin(i18n.Requirelogin)
self.config.log_last_value = None
check_value.check(self.config, str, 'Board', board)
check_value.check(self.config,
int,
'push_type',
push_type,
value_class=data_type.push_type)
check_value.check(self.config, str, 'PushContent', push_content)
if post_aid is not None:
check_value.check(self.config, str, 'PostAID', post_aid)
check_value.check(self.config, int, 'PostIndex', post_index)
if len(board) == 0:
raise ValueError(
log.merge(self.config,
[i18n.Board, i18n.ErrorParameter, board]))
if post_index != 0 and isinstance(post_aid, str):
raise ValueError(
log.merge(self.config, [
'PostIndex', 'PostAID', i18n.ErrorParameter, i18n.BothInput
]))
if post_index == 0 and post_aid is None:
raise ValueError(
log.merge(self.config, [
'PostIndex', 'PostAID', i18n.ErrorParameter, i18n.NoInput
]))
if post_index != 0:
newest_index = self._get_newest_index(data_type.index_type.BBS,
board=board)
check_value.check_index(self.config, 'PostIndex', post_index,
newest_index)
self._check_board(board)
max_push_length = 33
push_list = []
temp_start_index = 0
temp_end_index = temp_start_index + 1
while temp_end_index <= len(push_content):
temp = ''
last_temp = None
while len(temp.encode('big5-uao', 'replace')) < max_push_length:
temp = push_content[temp_start_index:temp_end_index]
if not len(temp.encode('big5-uao',
'replace')) < max_push_length:
break
elif push_content.endswith(temp):
break
elif temp.endswith('\n'):
break
elif last_temp == temp:
break
temp_end_index += 1
last_temp = temp
push_list.append(temp.strip())
temp_start_index = temp_end_index
temp_end_index = temp_start_index + 1
push_list = filter(None, push_list)
for push in push_list:
log.show_value(self.config, log.level.INFO, i18n.Push, push)
for _ in range(2):
try:
self._push(board,
push_type,
push,
post_aid=post_aid,
post_index=post_index)
break
except exceptions.NoFastPush:
# screens.show(self.config, self.connect_core.getScreenQueue())
log.log(self.config, log.level.INFO, '等待快速推文')
time.sleep(5.2)
def data_recv_An24(sock):
# ************
# connect An24(bd_addr) then recieve data from it
# ************
#sock=bluetooth.BluetoothSocket( bluetooth.RFCOMM )
#sock.connect((bd_addr, port))
global check_value
#sock = init_An24.conn(bd_addr)
#print '[OK] connecting '
#print 'prepare check...'
#init_An24.checking(sock)
'''
while init_An24.checking(sock) != [0, 0, 0, 0, 0]:
#check_value = init_check(check_signal(sock))
check_value = init_An24.checking(sock)
print '[not ok] check', check_value
time.sleep(5)
check_value = [0, 0, 0, 0, 0]
print '[ok] check', check_value
'''
#print '[start...]'
sock = init_An24.start(sock)
#print '[waiting reply...]'
#print '[ok] ready to accept data '
#output = open('output.txt','a')
pattern = re.compile(r'1002.*?1003', re.DOTALL)
lbuf = ''
endstr = '1003'
endpos = 0
data_one_sec = []
while 1:
buf = sock.recv(65535)
if not len(buf):
break
hexbuf = buf.encode('hex')
lbuf = lbuf + hexbuf
#regbuf = pattern.findall(lbuf)
for m in pattern.finditer(lbuf):
log('source data:', m.group())
log('electrode:', init_An24.check_value)
data_one_sec = data_parse(m.group())
stream_in_cache(data_one_sec)
#print data_cache
#print stream_in_cache()
#print type(data_all) #tuple
#print len(data_all)
#print type(m.group()) # //str
#print len(m.group()) # //76
#print m.group().isdigit() #//False
#print dir(m.group())
if endstr in lbuf:
endpos = lbuf.index(endstr) + 4
else:
pass
lbuf = lbuf[endpos:]
close_data_thread()
output.close()
sock.close()
def get_post(self,
board: str,
post_aid: str = None,
post_index: int = 0,
search_type: int = 0,
search_condition: str = None,
query: bool = False) -> data_type.PostInfo:
self._one_thread()
if not self._login_status:
raise exceptions.Requirelogin(i18n.Requirelogin)
self.config.log_last_value = None
check_value.check(self.config, str, 'Board', board)
if post_aid is not None:
check_value.check(self.config, str, 'PostAID', post_aid)
check_value.check(self.config, int, 'PostIndex', post_index)
check_value.check(self.config,
int,
'SearchType',
search_type,
value_class=data_type.post_search_type)
if search_condition is not None:
check_value.check(self.config, str, 'SearchCondition',
search_condition)
if len(board) == 0:
raise ValueError(
log.merge(self.config,
[i18n.Board, i18n.ErrorParameter, board]))
if post_index != 0 and isinstance(post_aid, str):
raise ValueError(
log.merge(self.config, [
'PostIndex', 'PostAID', i18n.ErrorParameter, i18n.BothInput
]))
if post_index == 0 and post_aid is None:
raise ValueError(
log.merge(self.config,
['PostIndex', 'PostAID', i18n.ErrorParameter]))
if search_condition is not None and search_type == 0:
raise ValueError(
log.merge(self.config, [
'SearchType',
i18n.ErrorParameter,
]))
if search_type == data_type.post_search_type.PUSH:
try:
S = int(search_condition)
except ValueError:
raise ValueError(
log.merge(self.config, [
'SearchCondition',
i18n.ErrorParameter,
]))
if not (-100 <= S <= 110):
raise ValueError(
log.merge(self.config, [
'SearchCondition',
i18n.ErrorParameter,
]))
if post_aid is not None and search_condition is not None:
raise ValueError(
log.merge(self.config, [
'PostAID',
'SearchCondition',
i18n.ErrorParameter,
i18n.BothInput,
]))
if post_index != 0:
newest_index = self._get_newest_index(
data_type.index_type.BBS,
board=board,
search_type=search_type,
search_condition=search_condition)
if post_index < 1 or newest_index < post_index:
raise ValueError(
log.merge(self.config, [
'PostIndex',
i18n.ErrorParameter,
i18n.OutOfRange,
]))
self._check_board(board)
for i in range(2):
need_continue = False
post = None
try:
post = self._get_post(board, post_aid, post_index, search_type,
search_condition, query)
except exceptions.ParseError as e:
if i == 1:
raise e
need_continue = True
except exceptions.UnknownError as e:
if i == 1:
raise e
need_continue = True
except exceptions.NoSuchBoard as e:
if i == 1:
raise e
need_continue = True
except exceptions.NoMatchTargetError as e:
if i == 1:
raise e
need_continue = True
if post is None:
need_continue = True
elif not post.pass_format_check:
need_continue = True
if need_continue:
log.log(self.config, log.level.DEBUG, 'Wait for retry repost')
time.sleep(0.1)
continue
break
return post
import discord
from discord.ext import commands
from discord_slash import cog_ext, SlashContext
from discord_slash.utils.manage_commands import create_option
from log import log
l = log()
class Moderation(commands.Cog,
description="Moderation tools for your server!"):
def __init__(self, bot):
self.bot = bot
# addrole:
@commands.command(
aliases=["ar"],
help="Add a role to a member.\nRequires Manage Roles permission.")
@commands.has_permissions(manage_roles=True)
async def addrole(self, ctx, member: discord.Member, role: discord.Role):
l.used(ctx)
await member.add_roles(role)
await ctx.send(f"{member.mention} got the {role} role.")
@cog_ext.cog_slash(name="addrole", description="Adds a role")
@commands.has_permissions(manage_roles=True)
async def _addrole(self, ctx: SlashContext, member: discord.Member,
role: discord.Role):
l.used(ctx)
await member.add_roles(role)
await ctx.send(f"{member.mention} got the {role} role.")
def log(self, msg: str) -> None:
self._one_thread()
log.log(self.config, log.level.INFO, msg)
def crawl_board(
self,
crawl_type: int,
post_handler,
board: str,
# BBS版本
start_index: int = 0,
end_index: int = 0,
start_aid: str = None,
end_aid: str = None,
search_type: int = 0,
search_condition: str = None,
query: bool = False,
# 網頁版本
start_page: int = 0,
end_page: int = 0) -> list:
self._one_thread()
self.config.log_last_value = None
check_value.check(self.config,
int,
'crawl_type',
crawl_type,
value_class=data_type.crawl_type)
check_value.check(self.config, str, 'Board', board)
if len(board) == 0:
raise ValueError(
log.merge(self.config,
[i18n.Board, i18n.ErrorParameter, board]))
if crawl_type == data_type.crawl_type.BBS:
if not self._login_status:
raise exceptions.Requirelogin(i18n.Requirelogin)
check_value.check(self.config, int, 'SearchType', search_type)
if search_condition is not None:
check_value.check(self.config, str, 'SearchCondition',
search_condition)
if start_aid is not None:
check_value.check(self.config, str, 'StartAID', start_aid)
if end_aid is not None:
check_value.check(self.config, str, 'EndAID', end_aid)
if (start_aid is not None or end_aid is not None) and \
(start_index != 0 or end_index != 0):
raise ValueError(
log.merge(
self.config,
['AID', 'Index', i18n.ErrorParameter, i18n.BothInput]))
if (start_aid is not None or end_aid is not None) and \
(search_condition is not None):
raise ValueError(
log.merge(self.config, [
'AID', 'SearchCondition', i18n.ErrorParameter,
i18n.BothInput
]))
if search_type == data_type.post_search_type.PUSH:
try:
S = int(search_condition)
except ValueError:
raise ValueError(
log.merge(self.config, [
'SearchCondition',
i18n.ErrorParameter,
]))
if not (-100 <= S <= 110):
raise ValueError(
log.merge(self.config, [
'SearchCondition',
i18n.ErrorParameter,
]))
if start_index != 0:
newest_index = self._get_newest_index(
data_type.index_type.BBS,
board=board,
search_type=search_type,
search_condition=search_condition)
check_value.check_index_range(self.config,
'start_index',
start_index,
'end_index',
end_index,
max_value=newest_index)
elif start_aid is not None and end_aid is not None:
start_index = self.get_post(board,
post_aid=start_aid,
query=True).index
end_index = self.get_post(board, post_aid=end_aid,
query=True).index
check_value.check_index_range(self.config, 'start_index',
start_index, 'end_index',
end_index)
else:
raise ValueError(
log.merge(self.config,
[i18n.ErrorParameter, i18n.NoInput]))
log.show_value(self.config, log.level.DEBUG, 'StartIndex',
start_index)
log.show_value(self.config, log.level.DEBUG, 'EndIndex', end_index)
error_post_list = []
del_post_list = []
if self.config.log_level == log.level.INFO:
PB = progressbar.ProgressBar(max_value=end_index -
start_index + 1,
redirect_stdout=True)
for index in range(start_index, end_index + 1):
for i in range(2):
need_continue = False
post = None
try:
post = self._get_post(
board,
post_index=index,
search_type=search_type,
search_condition=search_condition,
query=query)
except exceptions.ParseError as e:
if i == 1:
raise e
need_continue = True
except exceptions.UnknownError as e:
if i == 1:
raise e
need_continue = True
except exceptions.NoSuchBoard as e:
if i == 1:
raise e
need_continue = True
except exceptions.NoMatchTargetError as e:
if i == 1:
raise e
need_continue = True
except exceptions.ConnectionClosed as e:
if i == 1:
raise e
log.log(self.config, log.level.INFO,
i18n.RestoreConnection)
self._login(self._ID, self._Password,
self.config.kick_other_login)
need_continue = True
except exceptions.UseTooManyResources as e:
if i == 1:
raise e
log.log(self.config, log.level.INFO,
i18n.RestoreConnection)
self._login(self._ID, self._Password,
self.config.kick_other_login)
need_continue = True
if post is None:
need_continue = True
elif not post.pass_format_check:
need_continue = True
if need_continue:
log.log(self.config, log.level.DEBUG,
'Wait for retry repost')
time.sleep(0.1)
continue
break
if self.config.log_level == log.level.INFO:
PB.update(index - start_index)
if post is None:
error_post_list.append(index)
continue
if not post.pass_format_check:
if post.aid is not None:
error_post_list.append(post.aid)
else:
error_post_list.append(index)
continue
if post.delete_status != data_type.post_delete_status.NOT_DELETED:
del_post_list.append(index)
post_handler(post)
if self.config.log_level == log.level.INFO:
PB.finish()
return error_post_list, del_post_list
else:
if self.config.host == data_type.host_type.PTT2:
raise exceptions.HostNotSupport(
lib_util.get_current_func_name())
# 網頁版本爬蟲
# https://www.ptt.cc/bbs/index.html
# 1. 取得總共有幾頁 MaxPage
newest_index = self._get_newest_index(data_type.index_type.WEB,
board=board)
# 2. 檢查 StartPage 跟 EndPage 有沒有在 1 ~ MaxPage 之間
check_value.check_index_range(self.config,
'StartPage',
start_page,
'EndPage',
end_page,
max_value=newest_index)
# 3. 把每篇文章(包括被刪除文章)欄位解析出來組合成 data_type.PostInfo
error_post_list = []
del_post_list = []
# PostAID = ""
_url = 'https://www.ptt.cc/bbs/'
index = str(newest_index)
if self.config.log_level == log.level.INFO:
PB = progressbar.ProgressBar(max_value=end_page - start_page +
1,
redirect_stdout=True)
def deleted_post(post_title):
if post_title.startswith('('):
if '本文' in post_title:
return data_type.post_delete_status.AUTHOR
elif post_title.startswith('(已被'):
return data_type.post_delete_status.MODERATOR
else:
return data_type.post_delete_status.UNKNOWN
else:
return data_type.post_delete_status.NOT_DELETED
for index in range(start_page, newest_index + 1):
log.show_value(self.config, log.level.DEBUG, 'CurrentPage',
index)
url = _url + board + '/index' + str(index) + '.html'
r = requests.get(url, cookies={'over18': '1'})
if r.status_code != requests.codes.ok:
raise exceptions.NoSuchBoard(self.config, board)
soup = BeautifulSoup(r.text, 'html.parser')
for div in soup.select('div.r-ent'):
web = div.select('div.title a')
post = {
'author':
div.select('div.author')[0].text,
'title':
div.select('div.title')[0].text.strip('\n').strip(),
'web':
web[0].get('href') if web else ''
}
if post['title'].startswith('('):
del_post_list.append(post['title'])
if post['title'].startswith('(本文'):
if '[' in post['title']:
post['author'] = post['title'].split(
'[')[1].split(']')[0]
else:
post['author'] = post['title'].split(
'<')[1].split('>')[0]
else:
post['author'] = post['title'].split('<')[1].split(
'>')[0]
post = data_type.PostInfo(
board=board,
author=post['author'],
title=post['title'],
web_url='https://www.ptt.cc' + post['web'],
delete_status=deleted_post(post['title']))
post_handler(post)
if self.config.log_level == log.level.INFO:
PB.update(index - start_page)
log.show_value(self.config, log.level.DEBUG, 'DelPostList',
del_post_list)
# 4. 把組合出來的 Post 塞給 handler
# 5. 顯示 progress bar
if self.config.log_level == log.level.INFO:
PB.finish()
return error_post_list, del_post_list
