def to_str(self, verbose = True):
string = self.itemclass
tx, ty = screen_coords_to_tile_coords(self.initial_x, self.initial_y)
string += " " + str(tx) + " " + str(ty)
if verbose:
log_message("Obj converted to string: " + string)
return string
def to_str(self, verbose = True):
string = VisibleObject.to_str(self, False)
if self.trigger != None:
string += " " + str(self.max_activations) + " " + self.trigger.trigger_type
if verbose:
log_message("Obj converted to string: " + string)
return string
def to_str(self, verbose = True):
string = DynamicObject.to_str(self, False)
try:
string += " " + str_from_dir(self.attached)
except:
pass
if verbose:
log_message("Obj converted to string: " + string)
return string
def do_GET(self):
soc=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
(scm,netloc,path,params,query,fragment)=urlparse.urlparse(self.path,'http')
###########解码url编码##################
query=urllib.unquote(query)
print "query:%s\n"%query.lower()
#############################
pairs=(scm,netloc,path,params,query,fragment)
print self.path
if not netloc:
netloc=self.headers.get('host',"")
if scm!='http' or fragment or not netloc:
self.send_error(400,"bad url %s"%self.path)
return
"""
sql注入检测
"""
ret=SQLFilter().detect(self,pairs)
if ret==False:
ProxyHandler.lock.acquire()
if ProxyHandler.buffer.get(self.client_address[0]):
ProxyHandler.buffer[self.client_address[0]]+=1
else:
ProxyHandler.buffer[self.client_address[0]]=0
if ProxyHandler.buffer[self.client_address[0]]>=5:
ProxyHandler.buffer.pop(self.client_address[0])
ProxyHandler.blacklist.append(self.client_address[0])
ProxyHandler.lock.release()
print "\t" "bye"
soc.close()
self.connection.close()
return
try:
if self._connect_to(netloc,soc):
log_message(self,'connectAccept.db') #记录成功连接日志
self.log_request()
soc.send("%s %s %s\r\n"%(self.command,urlparse.urlunparse(('','',path,params,query,'')),
self.request_version))
self.headers['Connection']='close'
del self.headers['Proxy-Connection']
for key_val in self.headers.items():
soc.send("%s: %s\r\n" %key_val)
soc.send("\r\n")
self._read_write(soc)
finally:
print "\t" "bye"
soc.close()
self.connection.close()
def spike_detection_job(DatFileNames, ProbeFileName, output_dir, output_name):
"""
Top level function that starts a data processing job.
"""
for DatFileName in DatFileNames:
if not os.path.exists(DatFileName):
raise Exception("Dat file %s does not exist" % DatFileName)
DatFileNames = [os.path.abspath(DatFileName)
for DatFileName in DatFileNames]
probe = probes.Probe(ProbeFileName)
n_ch_dat = Parameters['NCHANNELS']
sample_rate = Parameters['SAMPLERATE']
high_frequency_factor = Parameters['F_HIGH_FACTOR']
set_globals_samples(sample_rate, high_frequency_factor)
Parameters['CHUNK_OVERLAP'] = int(
sample_rate * Parameters['CHUNK_OVERLAP_SECONDS'])
Parameters['N_CH'] = probe.num_channels
max_spikes = Parameters['MAX_SPIKES']
basename = basenamefolder = output_name
# OutDir = join(output_dir, basenamefolder)
OutDir = output_dir
with indir(OutDir):
# Create a log file
GlobalVariables['log_fd'] = open(basename + '.log', 'w')
if Parameters['DEBUG']:
GlobalVariables['debug_fd'] = open(basename + '.debug', 'w')
Channels_dat = np.arange(probe.num_channels)
# Print Parameters dictionary to .log file
log_message("\n".join(["{0:s} = {1:s}".format(key, str(value))
for key, value in sorted(Parameters.iteritems()) if not key.startswith('_')]))
spike_detection_from_raw_data(basename, DatFileNames, n_ch_dat,
Channels_dat, probe.channel_graph,
probe, max_spikes)
numwarn = GlobalVariables['warnings']
if numwarn:
log_message(
'WARNINGS ENCOUNTERED: ' + str(numwarn) + ', check log file.')
# Close the log file at the end.
if 'log_fd' in GlobalVariables:
GlobalVariables['log_fd'].close()
def detect(self,owner,data):
(scm,netloc,path,params,query,fragment)=data
text=self._ParseData(query)
ret=re.search(':',text)
if ret:
if re.search('http://',text)==None:
return False
for item in self.patterns:
ret=item.search(text)
if ret!=None:
log_message(owner,'connectDeny.db',*(text,item.pattern))
print "can not pass pattern:\t%s\n"% item.pattern
return False
return True
def save(self):
conffile = codecs.open(data.levelpath(self.level_name), "w", "utf_8")
string = self.to_str()
log_message('Level data to save:')
log_message(string)
log_message('Saving level to ' + data.levelpath(self.level_name))
conffile.write(string)
conffile.close()
log_message('Level saved.')
return
def parse_config():
for world in WORLDS:
Variables.vdict["unlocked" + world] = 0
Variables.vdict["hiscore" + world] = 0
Variables.vdict["besttime" + world] = 0
Variables.vdict["sound"] = True
Variables.vdict["dialogue"] = True
Variables.vdict["verbose"] = False
Variables.vdict["fullscreen"] = False
file_path = os.path.join(get_config_path(), "config.txt")
try:
conffile = codecs.open(file_path, "r", "utf_8")
for line in conffile:
if line.strip() != "":
values = line.split("\t")
if values[0] == "unlocked":
try:
Variables.vdict["unlocked" + values[1]] = int(values[2])
except:
Variables.vdict["unlocked" + WORLDS[0]] = int(values[1]) #Old style config file compatibility
elif values[0] == "hiscore":
try:
Variables.vdict["hiscore" + values[1]] = int(values[2])
except:
Variables.vdict["hiscore" + WORLDS[0]] = int(values[1]) #Old style config file compatibility
elif values[0] == "besttime":
Variables.vdict["besttime" + values[1]] = int(values[2])
elif values[0] == "sound":
Variables.vdict["sound"] = str_to_bool(values[1])
elif values[0] == "dialogue":
Variables.vdict["dialogue"] = str_to_bool(values[1])
elif values[0] == "fullscreen":
Variables.vdict["fullscreen"] = str_to_bool(values[1])
except:
if write_config():
log_message("Created configuration file to " + file_path)
return
def parse_config():
"""
This parses a config file stored in the location given by get_config_path().
The parsed values are stored in the variables class of the variables module.
"""
for world in WORLDS:
variables['unlocked' + world] = 0
variables['hiscore' + world] = 0
variables['besttime' + world] = 0
variables['sound'] = True
variables['dialogue'] = True
variables['verbose'] = False
variables['fullscreen'] = False
variables['character'] = 0
file_path = os.path.join(get_config_path(), 'config.txt')
try:
conffile = codecs.open(file_path, 'r', 'utf_8')
for line in conffile:
line = line.strip()
if line == '':
continue
values = line.split('\t')
if values[0] == 'unlocked':
variables.vdict['unlocked' + values[1]] = int(values[2])
elif values[0] == 'hiscore':
variables.vdict['hiscore' + values[1]] = int(values[2])
elif values[0] == 'besttime':
variables.vdict['besttime' + values[1]] = int(values[2])
elif values[0] == 'sound':
variables.vdict['sound'] = str_to_bool(values[1])
elif values[0] == 'dialogue':
variables.vdict['dialogue'] = str_to_bool(values[1])
elif values[0] == 'fullscreen':
variables.vdict['fullscreen'] = str_to_bool(values[1])
elif values[0] == 'character':
variables.vdict['character'] = int(values[1])
except Exception:
if write_config():
log_message('Created configuration file to ' + file_path)
def change(self, change):
"""Apply a change to the level data according to a Change class object."""
if change == None:
return
log_message("Made change " + change.tile_change + " to coords " + str(change.coords[0]) + ", " + str(change.coords[1]))
if (change.tile_change == "remove"):
self.remove_tile(change.coords)
elif (change.tile_change == "save"):
self.save()
elif (change.tile_change == "W") or (change.tile_change == "B") or (change.tile_change == "S"):
self.remove_tile(change.coords)
change.coords = (change.coords[0] + FULL_TILES_HOR - TILES_HOR, change.coords[1] + FULL_TILES_VER - TILES_VER)
self.add_tile(change.tile_change, change.coords)
self.reset_active_tiles()
return
def extract_spikes(h5s, basename, DatFileNames, n_ch_dat,
ChannelsToUse, ChannelGraph,
max_spikes=None):
# some global variables we use
CHUNK_SIZE = Parameters['CHUNK_SIZE']
CHUNKS_FOR_THRESH = Parameters['CHUNKS_FOR_THRESH']
DTYPE = Parameters['DTYPE']
CHUNK_OVERLAP = Parameters['CHUNK_OVERLAP']
N_CH = Parameters['N_CH']
S_JOIN_CC = Parameters['S_JOIN_CC']
S_BEFORE = Parameters['S_BEFORE']
S_AFTER = Parameters['S_AFTER']
THRESH_SD = Parameters['THRESH_SD']
THRESH_SD_LOWER = Parameters['THRESH_SD_LOWER']
# filter coefficents for the high pass filtering
filter_params = get_filter_params()
print filter_params
progress_bar = ProgressReporter()
#m A code that writes out a high-pass filtered version of the raw data (.fil file)
fil_writer = FilWriter(DatFileNames, n_ch_dat)
# Just use first dat file for getting the thresholding data
with open(DatFileNames[0], 'rb') as fd:
# Use 5 chunks to figure out threshold
DatChunk = get_chunk_for_thresholding(fd, n_ch_dat, ChannelsToUse,
num_samples(DatFileNames[0],
n_ch_dat))
FilteredChunk = apply_filtering(filter_params, DatChunk)
# get the STD of the beginning of the filtered data
if Parameters['USE_HILBERT']:
first_chunks_std = np.std(FilteredChunk)
print 'first_chunks_std', first_chunks_std, '\n'
else:
if Parameters['USE_SINGLE_THRESHOLD']:
ThresholdSDFactor = np.median(np.abs(FilteredChunk))/.6745
else:
ThresholdSDFactor = np.median(np.abs(FilteredChunk), axis=0)/.6745
Threshold = ThresholdSDFactor*THRESH_SD
print 'Threshold = ', Threshold, '\n'
Parameters['THRESHOLD'] = Threshold #Record the absolute Threshold used
# set the high and low thresholds
do_pickle = False
if Parameters['USE_HILBERT']:
ThresholdStrong = Parameters['THRESH_STRONG']
ThresholdWeak = Parameters['THRESH_WEAK']
do_pickle = True
elif Parameters['USE_COMPONENT_ALIGNFLOATMASK']:#to be used with a single threshold only
ThresholdStrong = Threshold
ThresholdWeak = ThresholdSDFactor*THRESH_SD_LOWER
do_pickle = True
if do_pickle:
picklefile = open("threshold.p","wb")
pickle.dump([ThresholdStrong,ThresholdWeak], picklefile)
threshold_outputstring = 'Threshold strong = ' + repr(ThresholdStrong) + '\n' + 'Threshold weak = ' + repr(ThresholdWeak)
log_message(threshold_outputstring)
n_samples = num_samples(DatFileNames, n_ch_dat)
spike_count = 0
for (DatChunk, s_start, s_end,
keep_start, keep_end) in chunks(DatFileNames, n_ch_dat, ChannelsToUse):
############## FILTERING ########################################
FilteredChunk = apply_filtering(filter_params, DatChunk)
# write filtered output to file
if Parameters['WRITE_FIL_FILE']:
fil_writer.write(FilteredChunk, s_start, s_end, keep_start, keep_end)
############## THRESHOLDING #####################################
# NEW: HILBERT TRANSFORM
if Parameters['USE_HILBERT']:
FilteredChunkHilbert = np.abs(signal.hilbert(FilteredChunk, axis=0) / first_chunks_std) ** 2
BinaryChunkWeak = FilteredChunkHilbert > ThresholdWeak
BinaryChunkStrong = FilteredChunkHilbert > ThresholdStrong
BinaryChunkWeak = BinaryChunkWeak.astype(np.int8)
BinaryChunkStrong = BinaryChunkStrong.astype(np.int8)
#elif Parameters['USE_COMPONENT_ALIGNFLOATMASK']:
else: # Usual method
#FilteredChunk = apply_filtering(filter_params, DatChunk) Why did you filter twice!!!???
if Parameters['USE_COMPONENT_ALIGNFLOATMASK']:
if Parameters['DETECT_POSITIVE']:
BinaryChunkWeak = FilteredChunk > ThresholdWeak
BinaryChunkStrong = FilteredChunk > ThresholdStrong
else:
BinaryChunkWeak = FilteredChunk < -ThresholdWeak
BinaryChunkStrong = FilteredChunk < -ThresholdStrong
BinaryChunkWeak = BinaryChunkWeak.astype(np.int8)
BinaryChunkStrong = BinaryChunkStrong.astype(np.int8)
else:
if Parameters['DETECT_POSITIVE']:
BinaryChunk = np.abs(FilteredChunk)>Threshold
else:
BinaryChunk = (FilteredChunk<-Threshold)
BinaryChunk = BinaryChunk.astype(np.int8)
# write filtered output to file
#if Parameters['WRITE_FIL_FILE']:
# fil_writer.write(FilteredChunk, s_start, s_end, keep_start, keep_end)
# print 'I am here at line 313'
############### FLOOD FILL ######################################
ChannelGraphToUse = complete_if_none(ChannelGraph, N_CH)
if (Parameters['USE_HILBERT'] or Parameters['USE_COMPONENT_ALIGNFLOATMASK']):
if Parameters['USE_OLD_CC_CODE']:
IndListsChunkOld = connected_components(BinaryChunkWeak,
ChannelGraphToUse, S_JOIN_CC)
IndListsChunk = [] #Final list of connected components. Go through all \weak' connected components
# and only include in final list if there are some samples that also exceed the strong threshold
# This method works better than connected_components_twothresholds.
for IndListWeak in IndListsChunkOld:
# embed()
# if sum(BinaryChunkStrong[zip(*IndListWeak)]) != 0:
i,j = np.array(IndListWeak).transpose()
if sum(BinaryChunkStrong[i,j]) != 0:
IndListsChunk.append(IndListWeak)
else:
IndListsChunk = connected_components_twothresholds(BinaryChunkWeak, BinaryChunkStrong,
ChannelGraphToUse, S_JOIN_CC)
BinaryChunk = 1 * BinaryChunkWeak + 1 * BinaryChunkStrong
else:
IndListsChunk = connected_components(BinaryChunk,
ChannelGraphToUse, S_JOIN_CC)
if Parameters['DEBUG']: #TO DO: Change plot_diagnostics for the HILBERT case
if Parameters['USE_HILBERT']:
plot_diagnostics_twothresholds(s_start,IndListsChunk,BinaryChunkWeak, BinaryChunkStrong,BinaryChunk,DatChunk,FilteredChunk,FilteredChunkHilbert,ThresholdStrong,ThresholdWeak)
elif Parameters['USE_COMPONENT_ALIGNFLOATMASK']:
plot_diagnostics_twothresholds(s_start,IndListsChunk,BinaryChunkWeak,BinaryChunkStrong,BinaryChunk,DatChunk,FilteredChunk,-FilteredChunk,ThresholdStrong,ThresholdWeak)#TODO: change HIlbert in plot_diagnostics_twothresholds
else:
plot_diagnostics(s_start,IndListsChunk,BinaryChunk,DatChunk,FilteredChunk,Threshold)
if Parameters['WRITE_BINFIL_FILE']:
fil_writer.write_bin(BinaryChunk, s_start, s_end, keep_start, keep_end)
#print len(IndListsChunk), 'len(IndListsChunk)'
############## ALIGN AND INTERPOLATE WAVES #######################
nextbits = []
if Parameters['USE_HILBERT']:
for IndList in IndListsChunk:
try:
wave, s_peak, sf_peak, cm, fcm = extract_wave_hilbert_new(IndList, FilteredChunk,
FilteredChunkHilbert,
S_BEFORE, S_AFTER, N_CH,
s_start, ThresholdStrong, ThresholdWeak)
s_offset = s_start + s_peak
sf_offset = s_start + sf_peak
if keep_start<=s_offset<keep_end:
spike_count += 1
nextbits.append((wave, s_offset, sf_offset, cm, fcm))
except np.linalg.LinAlgError:
s = '*** WARNING *** Unalignable spike discarded in chunk {chunk}.'.format(
chunk=(s_start, s_end))
log_warning(s)
except InterpolationError:
s = '*** WARNING *** Interpolation error in chunk {chunk}.'.format(
chunk=(s_start, s_end))
log_warning(s)
# and return them in time sorted order
nextbits.sort(key=lambda (wave, s, s_frac, cm, fcm): s_frac)
for wave, s, s_frac, cm, fcm in nextbits:
uwave = get_padded(DatChunk, int(s)-S_BEFORE-s_start,
int(s)+S_AFTER-s_start).astype(np.int32)
# cm = add_penumbra(cm, ChannelGraphToUse,
# Parameters['PENUMBRA_SIZE'])
# fcm = get_float_mask(wave, cm, ChannelGraphToUse,
# 1.)
yield uwave, wave, s, s_frac, cm, fcm
# unfiltered wave,wave, s_peak, ChMask, FloatChMask
elif Parameters['USE_COMPONENT_ALIGNFLOATMASK']:
for IndList in IndListsChunk:
try:
if Parameters['DETECT_POSITIVE']:
wave, s_peak, sf_peak, cm, fcm, comp_normalised, comp_normalised_power = extract_wave_twothresholds(IndList, FilteredChunk,
FilteredChunk,
S_BEFORE, S_AFTER, N_CH,
s_start, ThresholdStrong, ThresholdWeak)
else:
wave, s_peak, sf_peak, cm, fcm,comp_normalised, comp_normalised_power = extract_wave_twothresholds(IndList, FilteredChunk,
-FilteredChunk,
S_BEFORE, S_AFTER, N_CH,
s_start, ThresholdStrong, ThresholdWeak)
s_offset = s_start+s_peak
sf_offset = s_start + sf_peak
if keep_start<=s_offset<keep_end:
spike_count += 1
nextbits.append((wave, s_offset, sf_offset, cm, fcm))
except np.linalg.LinAlgError:
s = '*** WARNING *** Unalignable spike discarded in chunk {chunk}.'.format(
chunk=(s_start, s_end))
log_warning(s)
except InterpolationError:
s = '*** WARNING *** Interpolation error in chunk {chunk}.'.format(
chunk=(s_start, s_end))
log_warning(s)
# and return them in time sorted order
nextbits.sort(key=lambda (wave, s, s_frac, cm, fcm): s_frac)
for wave, s, s_frac, cm, fcm in nextbits:
uwave = get_padded(DatChunk, int(s)-S_BEFORE-s_start,
int(s)+S_AFTER-s_start).astype(np.int32)
# cm = add_penumbra(cm, ChannelGraphToUse,
# Parameters['PENUMBRA_SIZE'])
# fcm = get_float_mask(wave, cm, ChannelGraphToUse,
# 1.)
yield uwave, wave, s, s_frac, cm, fcm
# unfiltered wave,wave, s_peak, ChMask, FloatChMask
else: #Original SpikeDetekt. This code duplication is regretable but probably easier to deal with
for IndList in IndListsChunk:
try:
wave, s_peak, sf_peak, cm = extract_wave(IndList, FilteredChunk,
S_BEFORE, S_AFTER, N_CH,
s_start,Threshold)
s_offset = s_start+s_peak
sf_offset = s_start + sf_peak
if keep_start<=s_offset<keep_end:
spike_count += 1
nextbits.append((wave, s_offset, sf_offset, cm))
except np.linalg.LinAlgError:
s = '*** WARNING *** Unalignable spike discarded in chunk {chunk}.'.format(
chunk=(s_start, s_end))
log_warning(s)
# and return them in time sorted order
nextbits.sort(key=lambda (wave, s, s_frac, cm): s_frac)
for wave, s, s_frac, cm in nextbits:
uwave = get_padded(DatChunk, int(s)-S_BEFORE-s_start,
int(s)+S_AFTER-s_start).astype(np.int32)
cm = add_penumbra(cm, ChannelGraphToUse,
Parameters['PENUMBRA_SIZE'])
fcm = get_float_mask(wave, cm, ChannelGraphToUse,
ThresholdSDFactor)
yield uwave, wave, s, s_frac, cm, fcm
# unfiltered wave,wave, s_peak, ChMask, FloatChMask
progress_bar.update(float(s_end)/n_samples,
'%d/%d samples, %d spikes found'%(s_end, n_samples, spike_count))
if max_spikes is not None and spike_count>=max_spikes:
break
progress_bar.finish()
def is_aligned(self):
aligned = self.rect.right % TILE_DIM == 0 and self.rect.bottom % TILE_DIM == 0
if not aligned:
log_message("tilepos " + str(self.rect.right) + " " + str(self.rect.bottom))
return aligned
def exceptions(e):
tb = traceback.format_exc()
print tb
timestamp = time.strftime('[%Y-%m-%d %H:%M:%S]')
log.log_message('%s Exception caught:\n%s' % (timestamp, tb))
return make_response(jsonify({'error':'Internal error'}), 500)
def after_request(response):
timestamp = time.strftime('[%Y-%m-%d %H:%M:%S]')
log.log_message('%s %s %s %s %s %s' % (
timestamp, request.remote_addr,request.method,
request.scheme, request.full_path, response.status))
return response
