def start(self, join=False):
self._t = threading.Thread(target=self._server.serve_forever)
self._t.setDaemon(True) # don't hang on exit
self._t.start()
log.info("Listening on %s", self.address)
if join:
self._t.join()
def queue_analysis(sentence, tags):
try:
db().execute(query(insert_new_analysis_sql(tags,sentence)))
log.info("Successfully appended new analysis to queue")
return { 'status' : 'OK' }
except Exception as e:
return protocol.error(e)
def force_blueprint_replication(self, cmp_name, cmp_version, blueprint_type='component'):
'''
when not using SOURCE_DML, will force replication of acs blueprint from SOURCE_DML to the current dml
'''
# nothing to do if target and source are the same
if urlparse.urlparse(self.dml_host).hostname == urlparse.urlparse(self.SOURCE_DML).hostname:
return
try:
# check if blueprint is missing targeted dml
for dml_repo in self.dml_repositories:
path = self.get_acs_blueprint_path(cmp_name, cmp_version, dml_repo, blueprint_type)
response = requests.head(path)
if response.status_code == requests.codes.ok:
# we force replication only if artifact is not found
return
for dml_repo in self.dml_repositories:
path = self.get_acs_blueprint_path(cmp_name, cmp_version, dml_repo, blueprint_type, self.SOURCE_DML)
response = requests.get(path)
if response.status_code == requests.codes.ok:
# artifact found in SOURCE_DML for this repo
break
except:
log.info(traceback.format_exc())
def loadWord2VecEmbeddings(filePath):
with open(filePath, 'rb') as file:
firstLine = file.readline()
embeddingsCount, embeddingSize = tuple(firstLine.split(' '))
embeddingsCount, embeddingSize = int(embeddingsCount), int(embeddingSize)
embeddingFormat = '{0}f'.format(embeddingSize)
wordIndexMap = {}
embeddings = []
log.info('Vocabulary size: {0}. Embedding size: {1}.', embeddingsCount, embeddingSize)
embeddingIndex = 0
while True:
word = ''
while True:
char = file.read(1)
if not char:
log.lineBreak()
return wordIndexMap, embeddings
if char == ' ':
word = word.strip()
break
word += char
embedding = struct.unpack(embeddingFormat, file.read(embeddingSize * 4))
wordIndexMap[word] = len(wordIndexMap)
embeddings.append(embedding)
embeddingIndex += 1
log.progress('Reading embeddings: {0:.3f}%.', embeddingIndex, embeddingsCount)
def run(self):
"""线程主函数
循环运行，接受新的客户端的连接。
"""
log.info('GPS server thread: start, port: %d' % self.gps_port)
try:
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind(('0.0.0.0', self.gps_port))
server.listen(1)
server.settimeout(3) # timeout: 3s
while self.running:
try:
conn, address = server.accept()
conn.settimeout(3)
self.got_client_cb(conn, address)
log.debug('new client from: %s' % str(address))
except socket.timeout:
pass
server.close()
log.info('GPS server thread: bye')
except Exception as e:
log.error('GPS server thread error: %s' % e)
self.running = False
def _store_single_variable_plots_(bg, bgII, pathbg, labelbg, fg, fgII,\
pathfg, labelfg, varname, varlabel, maxvalue=None, minvalue=None, nbins=100, ymax1=None, ymax2=None, ylab=""):
if maxvalue is None:
maxvalue = max([ _calc_max_val_(bg.d[varname]), _calc_max_val_(fg.d[varname]),
_calc_max_val_(bgI.d[varname]), _calc_max_val_(fgI.d[varname]),
_calc_max_val_(bgII.d[varname]), _calc_max_val_(fgII.d[varname]) ])
logging.info("plotting for %s with label=%s maxval=%f" % (varname,varlabel,maxvalue) )
#M_ll ANALYSIS:
analysis.plot_signal_background(x=bg.d[varname], wx=bg.w, \
y=fg.d[varname], wy=fg.w, \
numbins=nbins, xmax=maxvalue, xmin=minvalue, ymax=ymax1, variable=varlabel,
title="Signal = $\sigma$["+labelfg+"] - $\sigma$["+labelbg+
"]\nBackground = $\sigma$["+labelbg+"]",
ylabel = ylab)
pyplot.savefig(pathfg+"_"+varname+"_SIGNAL.png")
#M_ll set II ANALYSIS:
analysis.plot_signal_background(x=bgII.d[varname], wx=bgII.w, \
y=fgII.d[varname], wy=fgII.w, \
numbins=nbins, xmax=maxvalue, xmin=minvalue, ymax=ymax2, variable=varlabel,
title="Signal = $\sigma$["+labelfg+"] - $\sigma$["+labelbg+
"]\nBackground = $\sigma$["+labelbg+"] (set II cuts)",
ylabel = ylab)
#pyplot.ylim(ylim1)
pyplot.savefig(pathfg+"_setII_"+varname+"_SIGNAL.png")
def make():
words = []
words += getSyntacticWordRelationsWords('res/Syntactic-Word-Relations/questions-words.txt')
words += getSATWords('res/SAT-Questions/SAT-package-V3.txt')
words += getSimLex999Words('res/SimLex-999/SimLex-999.txt')
words += getWordSimilarity353Words('res/WordSimilarity-353/combined.csv')
words += getRubensteinGoodenoughWords('res/RG/EN-RG-65.txt')
words = list(set(words))
words = sorted(words)
log.info('Found {0} words.', len(words))
whiteListPath = 'res/Tools/white_list.txt'
if os.path.exists(whiteListPath):
os.remove(whiteListPath)
with open(whiteListPath, 'w+') as whiteListFile:
batchSize = 10
batchesCount = len(words) / batchSize + 1
for batchIndex in xrange(0, batchesCount):
batch = words[batchIndex * batchSize : (batchIndex + 1) * batchSize]
line = ' '.join(batch) + '\n'
line = line.lower()
whiteListFile.write(line)
log.progress('Saving white list: {0:.0f}%.', batchIndex + 1, batchesCount)
log.lineBreak()
log.info('White list has been saved.')
def main():
if len(sys.argv) > 1:
mode=sys.argv[1]
else:
mode='sim'
defaults.add_epsilon_as_inset = True
sim = TriHoles2D(
material='SiN',
radius=0.34,
numbands=4,#8,
k_interpolation=5,#31,
resolution=16,
mesh_size=7,
runmode=mode,
num_processors=2,
save_field_patterns=False,
convert_field_patterns=False)
if not sim:
log.error('an error occurred during simulation. See the .out file')
return
log.info(' ##### success! #####\n\n')
def discorver_new_file(self,new_file):
'''
上传新文件到远端
'''
time.sleep(3)
log.info(new_file)
pass
def userFunction (self):
print "------ MainFileClassA: userFunction"
error( ' -- MainFileClass userFunction: Error logging\n' )
warn( ' -- MainFileClass userFunction: warn loggging\n' )
debug( ' -- MainFileClass userFunction: debug logging\n' )
info( ' -- MainFileClass userFunction: info logging\n' )
output( ' -- MainFileClass userFunction: output logging\n' )
def main():
if len(sys.argv) > 1:
mode=sys.argv[1]
else:
mode='sim'
sim = TriHolesSlab3D(
material='Si',
substrate_material='glass',
radius=0.5 * 367 / pitch,
thickness=116 / pitch,
numbands=16,
k_interpolation=31,
resolution=32,
mesh_size=7,
supercell_z=10,
runmode=mode,
num_processors=8,
save_field_patterns=False,
convert_field_patterns=False,
job_name_suffix='_on_glass',
bands_title_appendix=' (on glass)',
modes=[''] # use (run), not (run-zeven) etc.
)
if not sim:
log.error('an error occurred during simulation. See the .out file')
else:
log.info(' ##### Si PhC slab on glass - success! #####\n\n')
def __init__ (self):
print "------ MainFileClassA __init__"
error( ' -- MainFileClassA __init__: Error logging\n' )
warn( ' -- MainFileClassA __init__: warn loggging\n' )
debug( ' -- MainFileClassA __init__: debug logging\n' )
info( ' -- MainFileClassA __init__: info logging\n' )
output( ' -- MainFileClassA __init__: output logging\n' )
def received_message(self, payload):
"""
Dispatcher : call the controller function matching on the 'type'
field of the incoming message, if any.
The arguments of the controller's function should match:
\code
def a_method(self, data, socket):
\endcode
\note the data are passed as they are received, it is up to you to
determine if it is json-encoded or if it needs a special treatment.
\note the controller will get a reference to this socket object in
case it needs to send back an answer to the client.
"""
payload = json.loads(str(payload))
log.info(self, "收到: %s" % payload)
# do nothing if no controller was set
if self.controller is None: return
# determine which controller's function to call
try:
callback = getattr(self.controller, payload.get("type"))
if callback is not None and hasattr(callback, '__call__'):
callback(payload.get('data'), self)
except AttributeError:
log.error(self, 'callback undefined')
def send_msg(self, message_type, message=""):
"""
Send a message to the client. Format : a dictionary with 'type' and 'data' keys.
The dictionary will be converted to json for the transport.
"""
self.send(json.dumps({'type': message_type, 'data': message}))
log.info(self, "发送 %s: %s" % (message_type, message ))
def _cache_fingerprint(self, userid, fpr, key):
"""
Caches the given fingerprint after having done some checks.
@return: the fingerprint if it was cached, None otherwise
"""
if self._fingerprints is None:
# we fake having cached the fingerprint in this case
return fpr
if userid in self._fingerprints:
oldfpr = self._fingerprints[userid]
if oldfpr != fpr:
# an old key was found, check for validity
oldkey = self.ctx.get_key(oldfpr)
# step 1: check for expiration/revocation
if self._check_key(userid, oldkey, oldfpr):
log.debug("old key is still valid")
# step 2: check for key start date
if key.subkeys[0].timestamp <= oldkey.subkeys[0].timestamp:
log.info("new key is older than old key (old: %d, new: %d)" % (oldkey.subkeys[0].timestamp,
key.subkeys[0].timestamp))
return None
self._fingerprints[userid] = fpr
return fpr
def discorver_new_file(self,new_file):
'''
本地持久化新文件
'''
time.sleep(3)
log.info(new_file)
pass
def _check_and_execute_one(self, judge, curtime):
if judge.is_matched(curtime):
log.info('Matched: "%s" -> "%s"' % \
(judge._query,judge.get_commandline())
)
self._execute_commandline(judge.get_commandline())
def buildPkg(pkgPath, pkgDestDir):
global buildContinue
global commands
global configOptions
global configBased
log.info(" building in " + pkgPath)
pwd = os.getcwd()
os.chdir(pkgPath)
if configBased:
if commands.has_key('configure') or len(configOptions) != 0:
runConfigScript(pkgPath, configOptions)
else:
runConfigParser(pkgPath)
runConfigScript(pkgPath, configOptions)
checkProgress()
if buildContinue:
runMake(pkgPath)
checkProgress()
if buildContinue:
runMakeInstallDestDir(pkgPath, pkgDestDir)
checkProgress()
os.chdir(pwd)
def _save(self):
"""Saves the last text to the file."""
try:
with open(self._save_filename, 'w') as f:
f.write(self._commits[-1][1].text)
except IOError:
log.info('Cannot save the text to the file.')
def __init__(pkgPath, pkgDestDir, optionsDict, commandsDict, verbosity_level, timeresources_show):
global VERBOSE
VERBOSE = verbosity_level
global TIMERESOURCES
TIMERESOURCES = timeresources_show
global commands
commands = commandsDict
global options
options = optionsDict
global commandsSet
log.setLog("infoLog")
if (cmp(pkgPath, "") != 0):
checkAppropriate(pkgPath)
runConfigBased(pkgPath)
if (heuristicApproved == True):
if VERBOSE:
log.info(" commencing package build with heuristic \"" + heuristicShortName + "\"")
if TIMERESOURCES:
log.info(" task started on "+ str(time.clock()))
buildPkg(pkgPath, pkgDestDir)
return commandsSet
def downloadComic(self, isTest = False, maxComics = sys.maxsize):
"""Download the comic strip.
:param isTest: Used to test that the ComicMiner is correctly finding the images and next urls without downloading any images
:param maxComix: Also used mostly in testing to limit the number of comics to go through before stopping
"""
#Start the requests session
self.startSession(self.BaseUrl)
#Count is used in the naming of the local image files and to prematurely stop the comic downloads
count = 0
#The first url to use is the first comic in the strips URL
nextUrl = self.StartUrl
while nextUrl and count < maxComics:
#Get the html from the nextUrl
htmlResponse = self.gotoUrl(self.BaseUrl + nextUrl, self.Session)
#Break if the html request fails
if not htmlResponse:
break
#Find the image url in the html
currentImageUrl = self.findComicImage(htmlResponse.text)
log.info ("Current Image Url: {0}".format(currentImageUrl))
if not isTest:
#Download the comic image from the image url
self.downloadComicImage(currentImageUrl, self.formatCount(count) + "-", self.StorageDir + self.ComicName + "/")
#Find the next url in the html
nextUrl = self.findNextUrl(htmlResponse.text)
log.info ("Next Url: {0}".format(nextUrl))
count += 1
def runMakeInstallDestDir(pkgPath, pkgDestDir):
global buildContinue
if (os.path.exists(pkgPath + r'/Makefile')):
log.info(" (R): make DESTDIR=" + pkgDestDir +" install")
runCommandConfirmation = None
question = " heuristic [automake]: Run suggested command? (Y/N): "
while runCommandConfirmation not in ('y','ye','yes','n','no'):
runCommandConfirmation = raw_input(question).lower()
if runCommandConfirmation not in ('y','ye','yes','n','no'):
log.info(" heuristic: Please enter Y or N")
if runCommandConfirmation.lower() == "y" or \
runCommandConfirmation.lower() == "yes" or \
runCommandConfirmation.lower() == "ye":
print 40*"-"
# run "make DESTDIR=$DIR install"
commandErrCode = subprocess.call("make DESTDIR=" + pkgDestDir + " install", shell=True)
print 40*"-"
if (commandErrCode != 0):
log.info(" (S): command failed (error code " + str(commandErrCode) + ")")
buildContinue = False
if (commandErrCode == 0):
log.info(" (R): success")
commandsSet.append("make DESTDIR=$TMP install")
else:
log.info(" (R): command not run")
def launch(pathTo, hyper):
fileIndexMap = parameters.loadMap(pathTo.textIndexMap)
filesCount = len(fileIndexMap)
fileEmbeddingSize = hyper.fileEmbeddingSize
wordIndexMap = parameters.loadMap(pathTo.wordIndexMap)
wordEmbeddings = parameters.loadEmbeddings(pathTo.wordEmbeddings)
metricsPath = pathTo.metrics('history.csv')
if os.path.exists(metricsPath):
os.remove(metricsPath)
contextProvider = parameters.IndexContextProvider(pathTo.contexts)
windowSize = contextProvider.windowSize - 1
contextSize = windowSize - 1
negative = contextProvider.negative
contexts = contextProvider[:]
log.info('Contexts loading complete. {0} contexts loaded {1} words and {2} negative samples each.',
len(contexts),
contextProvider.windowSize,
contextProvider.negative)
fileEmbeddings = rnd2(filesCount, fileEmbeddingSize)
model = Model(fileEmbeddings, wordEmbeddings, contextSize=contextSize, negative=negative)
# model = Model.load(pathTo.fileEmbeddings, pathTo.wordEmbeddings, pathTo.weights)
train(model, fileIndexMap, wordIndexMap, wordEmbeddings, contexts,
epochs=hyper.epochs,
batchSize=hyper.batchSize,
learningRate=hyper.learningRate,
metricsPath=metricsPath)
model.dump(pathTo.fileEmbeddings, pathTo.weights)
def _build_import_library_x86():
""" Build the import libraries for Mingw32-gcc on Windows
"""
lib_name = "python%d%d.lib" % tuple(sys.version_info[:2])
lib_file = os.path.join(sys.prefix,'libs',lib_name)
out_name = "libpython%d%d.a" % tuple(sys.version_info[:2])
out_file = os.path.join(sys.prefix,'libs',out_name)
if not os.path.isfile(lib_file):
log.warn('Cannot build import library: "%s" not found' % (lib_file))
return
if os.path.isfile(out_file):
log.debug('Skip building import library: "%s" exists' % (out_file))
return
log.info('Building import library (ARCH=x86): "%s"' % (out_file))
from numpy.distutils import lib2def
def_name = "python%d%d.def" % tuple(sys.version_info[:2])
def_file = os.path.join(sys.prefix,'libs',def_name)
nm_cmd = '%s %s' % (lib2def.DEFAULT_NM, lib_file)
nm_output = lib2def.getnm(nm_cmd)
dlist, flist = lib2def.parse_nm(nm_output)
lib2def.output_def(dlist, flist, lib2def.DEF_HEADER, open(def_file, 'w'))
dll_name = "python%d%d.dll" % tuple(sys.version_info[:2])
args = (dll_name,def_file,out_file)
cmd = 'dlltool --dllname %s --def %s --output-lib %s' % args
status = os.system(cmd)
# for now, fail silently
if status:
log.warn('Failed to build import library for gcc. Linking will fail.')
#if not success:
# msg = "Couldn't find import library, and failed to build it."
# raise DistutilsPlatformError, msg
return
def __path_components(self, path):
"""
Splits a path into a list of path components relative to the configuration file.
For example, if the manifest file was at /tmp/exile.manifest:
/tmp/path/to/a/file
becomes:
['path', 'to', 'a', 'file']
"""
path = os.path.realpath(path)
if not path.startswith(self.__root):
if not self.__silent:
log.info("skipping path outside manifest scope: " + path)
return None
relative = os.path.relpath(path, self.__root)
parts = []
head, tail = os.path.split(relative)
while tail:
parts = [tail] + parts
head, tail = os.path.split(head)
if head:
parts = [head] + parts
return parts
def runOnSchedule():
global sched_pump_start, sched_pump_stop, sched_sweep_start, sched_sweep_stop
if _inZone(sched_sweep_start, sched_sweep_stop):
if state() != STATE_SCHEDULED_SWEEP:
log.info("Starting Scheduled Sweep Run")
startSweep()
_setState(STATE_SCHEDULED_SWEEP)
return True
if _inZone(sched_pump_start, sched_pump_stop):
if state() != STATE_SCHEDULED_PUMP:
log.info("Starting Scheduled Pump Run")
startPump()
_setState(STATE_SCHEDULED_PUMP)
return True
if solar.runPumpsIfNeeded():
return False
if state() in [STATE_SCHEDULED_PUMP, STATE_SCHEDULED_SWEEP]:
log.info("Stopping scheduled run")
stopAll()
return False
if state() in [STATE_SOLAR, STATE_SOLAR_MIXING]:
log.info("Stopping pumps for solar heating")
stopAll()
return False
if state() != STATE_OFF and getStartTime() < time.time() - RUN_TIME:
log.info("Stopping manual run")
stopAll()
return False
def init(self, config=None):
app_name = getattr(config, 'app_name', None)
log_path = getattr(config, 'log_path', None)
pidfile = getattr(config, 'pidfile', None)
if not app_name:
app_name = os.environ.get('APP_NAME', 'std')
if not log_path:
log_path = os.environ.get('LOG_PATH', './log')
if not pidfile:
pidfile = os.environ.get('PIDFILE', './pidfile')
log_path = os.path.abspath(log_path)
pidfile = os.path.abspath(pidfile)
if not os.path.exists(log_path):
os.makedirs(log_path)
os.environ['LOG_PATH'] = log_path
config.log_path = log_path
config.pidfile = pidfile
log.info('log_path:%s', log_path)
self.stdin_path = join(log_path, '%s_in.log' % app_name)
self.stdout_path = join(log_path, '%s_out.log' % app_name)
self.stderr_path = join(log_path, '%s_err.log' % app_name)
pid_path = os.path.dirname(pidfile)
if not os.path.exists(pid_path):
os.makedirs(pid_path)
self.pidfile_path = pidfile
self.pidfile_timeout = 3
with open(self.stdin_path, 'a'):
pass
def read(path):
"""Parse a CSV flat file into an ordered dictionary."""
parks = collections.OrderedDict()
log.info("reading %s...", path)
with open(path, 'r') as csvfile:
rows = csv.reader(csvfile)
header = None
for row in rows:
if not row:
continue
if header is None:
header = row
log.debug("header: %s", header)
else:
log.debug("row: %s", row)
data = AttributeDictionary()
name = row[0]
for index, key in enumerate(header):
data[key] = row[index]
parks[name] = data
log.info("read %s parks", len(parks))
return parks
def wa_msg_received(self, message):
store_msg(message, self.log_file)
info(" <<< Received WA message: %s" %message)
if message.chan == self.wa_phone:
#private message
if message.target is None:
# directed to bot itself
nick = self.contacts[message.get_nick()]
irc_msg = "<%s> %s" %(nick, message.msg)
irc_target = self.contacts[message.nick_full.split("@")[0]]
self.irc_i.send(self.owner_nick, irc_msg)
else:
# directed to someone
try:
phone = message.get_nick()
nick = self.contacts[phone]
target = self.get_group_from_chan(self.contacts, message.target)
msg = "<%s> %s" %(target, message.msg)
self.irc_i.send(target, msg)
except:
error("Couldn't relay directed WA msg to IRC")
else:
#group message
try:
msg = "<%s> %s" %(self.contacts[message.get_nick()], message.msg)
except:
error("Contact not recognized")
msg = "<%s> %s" %(message.get_nick(), message.msg)
try:
self.irc_i.send(self.contacts[message.chan], msg)
except:
error("Channel %s not recognized" %(message.chan))
def set_controller(self, controller):
"""
Set the controller used by this socket.
"""
self.controller = controller
self.controller.register_client_socket(self)
log.info(self, 'client socket registered')
def reply(message):
if vk.getSender(message) in banign.banned:
return None
if vk.getSender(message) < 0:
return None
if vk.getSender(message) in check_friend.noadd or message[
'user_id'] in check_friend.noadd:
return ('', 0)
if 'deactivated' in vk.users[message['user_id']] or vk.users[
message['user_id']]['blacklisted'] or vk.users[
message['user_id']]['blacklisted_by_me']:
return ('', 0)
if 'body' not in message:
message['body'] = ''
if 'id' not in message: # friendship request
return (getBotReply(message['user_id'], message['message'], 0,
'friendship request'), 2)
message['body'] = preprocessMessage(message)
if message['body']:
if message['body'].startswith('\\') and len(message['body']) > 1:
cmd = message['body'][1:].split()
if cmd and message['user_id'] == vk.admin:
return (processCommand(*cmd), 1)
if isBotMessage(message['body']):
text_msg = '({}) {} - ignored (bot message)'.format(
vk.printableSender(message, False), message['body'])
html_msg = '({}) {} - ignored (bot message)'.format(
vk.printableSender(message, True), message['body'])
log.info((text_msg, html_msg))
if 'chat_id' in message:
bot_users[message['user_id']] = bot_users.get(
message['user_id'], 0) + 1
if bot_users[message['user_id']] >= 3:
log.info('Too many bot messages')
log.write(
'conf',
str(message['user_id']) + ' ' +
str(message['chat_id']) + ' (bot messages)')
vk.leaveConf(message['chat_id'])
return ('', 0)
elif message['user_id'] in bot_users:
del bot_users[message['user_id']]
if message['body'] == last_message_text.get(
message['user_id'], (0, 0, 0))[0] and message['body'] != '..':
last_message_text[message['user_id']][2] += 1
if last_message_text[message['user_id']][2] >= 5:
noaddUsers([message['user_id']], reason='flood')
else:
text_msg = '({}) {} - ignored (repeated)'.format(
vk.printableSender(message, False), message['body'])
html_msg = '({}) {} - ignored (repeated)'.format(
vk.printableSender(message, True), message['body'])
log.info((text_msg, html_msg))
return ('', 0)
if message['body'].strip().upper() == last_message_text.get(
message['user_id'], (0, 0, 0))[1]:
text_msg = '({}) {} - ignored (my reply)'.format(
vk.printableSender(message, False), message['body'])
html_msg = '({}) {} - ignored (my reply)'.format(
vk.printableSender(message, True), message['body'])
log.info((text_msg, html_msg))
return ('', 0)
t = evalExpression(message['body'])
if t:
if getBotReply(None, message['body'], -2):
return ('', 0)
text_msg = '({}) {} = {} (calculated)'.format(
vk.printableSender(message, False), message['body'], t)
html_msg = '({}) {} = {} (calculated)'.format(
vk.printableSender(message, True), message['body'], t)
log.info((text_msg, html_msg))
log.write(
'calc', '{}: "{}" = {}'.format(message['user_id'],
message['body'], t))
return (t, 0)
if message['body']:
message['body'] = message['body'].replace('<br>', '<BR>')
if message['body'] and message['body'].upper() == message['body'] and len(
[i for i in message['body'] if i.isalpha()]) > 1:
text_msg = '({}) {} - ignored (caps)'.format(
vk.printableSender(message, False), message['body'])
html_msg = '({}) {} - ignored (caps)'.format(
vk.printableSender(message, True), message['body'])
log.info((text_msg, html_msg))
return ('', 0)
reply = getBotReply(message['user_id'], message['body'],
message.get('chat_id', 0), message.get('_method', ''))
if reply is not None:
last_message_text[message['user_id']] = [
message['body'], reply.strip().upper(), 1
]
return (reply, 0)
def getBotReply(uid, message, conf_id, method=''):
if message is None:
return None
message = message.replace('\u0401', '\u0415').replace('\u0451',
'\u0435') # yo
message = message.replace('\u0490', '\u0413').replace('\u0491',
'\u0433') # g
message = message.replace('\u0404', '\u042d').replace('\u0454',
'\u044d') # e
message = message.replace('\u0406', '\u0418').replace('\u0456',
'\u0438') # i
message = message.replace('\u0407', '\u0418').replace('\u0457',
'\u0438') # i
message = message.replace("`", "'")
if conf_id == 0:
answer = bot.interact('user {} {}'.format(uid, message))
elif conf_id > 0:
answer = bot.interact('conf {} {}'.format(uid, message))
elif conf_id in (-1, -2):
answer = bot.interact('{} {}'.format(
'comm' if conf_id == -1 else 'flat 0', message))
bl = (answer == '$blacklisted')
return bl
if answer == '$noans':
if conf_id > 0:
answer = ''
else:
if message.upper() == message.lower() and '?' not in message:
answer = random.choice(smiles)
else:
answer = noans[0]
next_ans = random.randint(1, len(noans) - 1)
noans[0], noans[next_ans] = noans[next_ans], noans[0]
elif answer == '$blacklisted':
answer = ''
if message == message.lower() and message != message.upper(
) or message.upper() == message.lower() and uid in _last_reply_lower:
_last_reply_lower.add(uid)
answer = answer.lower()
else:
_last_reply_lower.discard(uid)
console_message = ''
if '{' in answer:
answer, gender = applyGender(answer, uid)
console_message += ' (' + gender + ')'
if '\\' in answer:
r = re.compile(r'\\[a-zA-Z]+')
res = r.sub(lambda m: preprocessReply(m.group(0)[1:], uid), answer)
console_message += ' (' + answer + ')'
answer = res
if method:
console_message += ' (' + method + ')'
text_msg = '({}) {} : {}{}'.format(
vk.printableSender({
'user_id': uid,
'chat_id': conf_id
}, False), message, renderSmile(answer), console_message)
html_msg = '({}) {} : {}{}'.format(
vk.printableSender({
'user_id': uid,
'chat_id': conf_id
}, True), message,
renderSmile(answer).replace('&', '&'), console_message)
log.info((text_msg, html_msg))
return answer
fp = open(lock_file, 'w')
single = False
for i in range(100):
try:
fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
time.sleep(5)
else:
single = True
break
if not single:
sys.exit(0)
with open(pid_file, 'w') as f:
f.write(str(os.getpid()))
log.info('Starting vkbot, pid ' + str(os.getpid()))
_bot_message = re.compile(r'^\(.+\)')
def isBotMessage(msg):
return _bot_message.match(msg.strip())
bot_users = {}
bot = cpp_bot()
noans = open(accounts.getFile('noans.txt'),
encoding='utf-8').read().splitlines()
smiles = open(accounts.getFile('smiles.txt'),
def OSCljscene(path, tags, args, source):
log.info("Aurora OSC : got /aurora/ljscene with value" + str(args[0]))
lj.WebStatus("aurora to virtual " + str(args[0]))
ljscene = args[0]
lj.ljscene(ljscene)
def _read_embeddings_from_text_file(
file_uri: str,
embedding_dim: int,
vocab: Vocabulary,
namespace: str = "tokens") -> torch.FloatTensor:
"""
Read pre-trained word vectors from an eventually compressed text file, possibly contained
inside an archive with multiple files. The text file is assumed to be utf-8 encoded with
space-separated fields: [word] [dim 1] [dim 2] ...
Lines that contain more numerical tokens than ``embedding_dim`` raise a warning and are skipped.
The remainder of the docstring is identical to ``_read_pretrained_embeddings_file``.
"""
tokens_to_keep = set(
vocab.get_index_to_token_vocabulary(namespace).values())
vocab_size = vocab.get_vocab_size(namespace)
embeddings = {}
# First we read the embeddings from the file, only keeping vectors for the words we need.
info("Reading pretrained embeddings from file")
with EmbeddingsTextFile(file_uri) as embeddings_file:
for line in embeddings_file:
token = line.split(' ', 1)[0]
if token in tokens_to_keep:
fields = line.rstrip().split(' ')
if len(fields) - 1 != embedding_dim:
# Sometimes there are funny unicode parsing problems that lead to different
# fields lengths (e.g., a word with a unicode space character that splits
# into more than one column). We skip those lines. Note that if you have
# some kind of long header, this could result in all of your lines getting
# skipped. It's hard to check for that here; you just have to look in the
# embedding_misses_file and at the model summary to make sure things look
# like they are supposed to.
warn(
"Found line with wrong number of dimensions (expected: %d; actual: %d): %s",
embedding_dim,
len(fields) - 1, line)
continue
vector = numpy.asarray(fields[1:], dtype='float32')
embeddings[token] = vector
if not embeddings:
raise ConfigurationError(
"No embeddings of correct dimension found; you probably "
"misspecified your embedding_dim parameter, or didn't "
"pre-populate your Vocabulary")
all_embeddings = numpy.asarray(list(embeddings.values()))
embeddings_mean = float(numpy.mean(all_embeddings))
embeddings_std = float(numpy.std(all_embeddings))
# Now we initialize the weight matrix for an embedding layer, starting with random vectors,
# then filling in the word vectors we just read.
info("Initializing pre-trained embedding layer")
embedding_matrix = torch.FloatTensor(vocab_size, embedding_dim).normal_(
embeddings_mean, embeddings_std)
num_tokens_found = 0
index_to_token = vocab.get_index_to_token_vocabulary(namespace)
for i in range(vocab_size):
token = index_to_token[i]
# If we don't have a pre-trained vector for this word, we'll just leave this row alone,
# so the word has a random initialization.
if token in embeddings:
embedding_matrix[i] = torch.FloatTensor(embeddings[token])
num_tokens_found += 1
else:
info(
"Token %s was not found in the embedding file. Initialising randomly.",
token)
info("Pretrained embeddings were found for %d out of %d tokens",
num_tokens_found, vocab_size)
return embedding_matrix
def handle_proposals(
table: element.Tag,
*,
election: Election,
precinct: Precinct,
district: Optional[District],
**_,
) -> Optional[Proposal]:
if table.get('class') != ['proposal']:
return None
# Parse category
category = None
td = table.find(class_='division')
if td:
log.debug(f'Parsing category from division: {td.text!r}')
category_name = helpers.clean_district_category(
helpers.titleize(td.text.split("PROPOSALS")[0]))
if category_name == "Authority":
log.warn('Assuming category is county')
category_name = "County"
category = DistrictCategory.objects.get(name=category_name)
else:
log.debug(f'Reusing category from previous district: {district}')
assert district
category = district.category
log.info(f'Parsed {category!r}')
assert category
# Parse district
if category.name == "State":
log.debug('Inferring district as state')
district = District.objects.get(category=category, name="Michigan")
elif category.name == "County":
log.debug('Inferring district as county')
district = precinct.county
elif category.name in {"Jurisdiction", "City", "Township", "Village"}:
log.debug('Inferring district as jurisdiction')
district = precinct.jurisdiction
else:
proposal_title = table.find(class_='proposalTitle').text
proposal_text = table.find(class_='proposalText').text
log.debug(f'Parsing district from title: {proposal_title!r}')
title = helpers.titleize(proposal_title)
if category.name in title:
district_name = title.split(category.name)[0].strip()
log.debug(f'Parsed district name: {district_name}')
district, created = District.objects.get_or_create(
category=category, name=district_name)
if created:
log.warn(f'Added missing district: {district}')
elif precinct.jurisdiction.name in proposal_text:
log.warn('Assuming district is jurisdiction from proposal')
district = precinct.jurisdiction
elif precinct.county.name in proposal_text:
log.warn('Assuming district is county from proposal')
district = precinct.county
elif " county " in proposal_title.lower():
district_name = helpers.titleize(
proposal_title.lower().split(" county ")[0].split('.')[-1])
log.warn(f'Assuming district is different county: {district_name}')
district = District.objects.get(category=precinct.county.category,
name=district_name)
elif category.name in {"Community College", "District Library"}:
log.warn(f'Assuming district is county from category')
district = precinct.county
else:
assert False, f'Could not determine district: {table}'
log.info(f'Parsed {district!r}')
assert district
# Parse proposal
proposal_title = table.find(class_='proposalTitle').text
proposal_text = table.find(class_='proposalText').text
log.debug(f'Parsing proposal from text: {proposal_text!r}')
proposal, _ = Proposal.objects.update_or_create(
election=election,
district=district,
name=helpers.titleize(proposal_title),
defaults=dict(description=proposal_text.strip()),
)
log.info(f'Parsed {proposal!r}')
# Add precinct
proposal.precincts.add(precinct)
proposal.save()
return proposal
def verify_services_status(self):
cps_service = CPSServiceBusiness()
for proxy in self.proxy_match_region.keys():
cps_service.check_all_service_template_status(proxy)
if __name__ == '__main__':
log.init('patches_tool_config')
config.export_env()
print(
'Start to patch Hybrid-Cloud patches in cascading node and proxy nodes...'
)
log.info(
'Start to patch Hybrid-Cloud patches in cascading node and proxy nodes...'
)
patches_tool = PatchesTool()
patches_tool.patch_for_cascading_and_proxy_node()
print(
'Finish to patch Hybrid-Cloud patches in cascading node and proxy nodes...'
)
log.info(
'Finish to patch Hybrid-Cloud patches in cascading node and proxy nodes...'
)
print('Start to patch Hybrid-Cloud patches in cascaded nodes...')
log.info('Start to patch Hybrid-Cloud patches in cascaded nodes...')
def analyze_move(board, move_col, move_row, move_mark):
result = MoveResult()
# Consecutive pieces of the same mark as the move
cons_left = 0
cons_right = 0
cons_up = 0
cons_down = 0
cons_nw = 0
cons_ne = 0
cons_sw = 0
cons_se = 0
# Check left (columns "move-1" to "0")
for i in range(0, move_col):
# Search from the move to the left
col = move_col - i - 1
if board[move_row][col] == move_mark:
#log.debug("Found consecutive mark at col: {} row: {}".format(col, move_row))
cons_left += 1
else:
break
# Check right (columns "move" to num_cols)
for i in range(move_col + 1, constants.num_cols):
# Search from the move to the right
col = i
if board[move_row][col] == move_mark:
#log.debug("Found consecutive mark at col: {} row: {}".format(col, move_row))
cons_right += 1
else:
break
# Check up
for i in range(0, move_row):
# Search from the move to the right
row = move_row - i - 1
if board[row][move_col] == move_mark:
#log.debug("Found consecutive mark at col: {} row: {}".format(move_col, row))
cons_up += 1
else:
break
# Check down
for i in range(move_row + 1, constants.num_rows):
# Search from the move to the right
row = i
if board[row][move_col] == move_mark:
#log.debug("Found consecutive mark at col: {} row: {}".format(move_col, row))
cons_down += 1
else:
break
# Check NW
row = move_row
for i in range(0, move_col):
col = move_col - i - 1
row = row - 1
if row < 0:
break
if board[row][col] == move_mark:
#log.debug("Found consecutive mark at col: {} row: {}".format(col, row))
cons_nw += 1
else:
break
# Check NE
row = move_row
for i in range(move_col + 1, constants.num_cols):
col = i
row = row - 1
if row < 0:
break
if board[row][col] == move_mark:
#log.debug("Found consecutive mark at col: {} row: {}".format(col, row))
cons_ne += 1
else:
break
# Check SW
row = move_row
for i in range(0, move_col):
col = move_col - i - 1
row = row + 1
if row >= constants.num_rows:
break
if board[row][col] == move_mark:
#log.debug("Found consecutive mark at col: {} row: {}".format(col, row))
cons_sw += 1
else:
break
# Check SE
row = move_row
for i in range(move_col + 1, constants.num_cols):
col = i
row = row + 1
if row >= constants.num_rows:
break
if board[row][col] == move_mark:
#log.debug("Found consecutive mark at col: {} row: {}".format(col, row))
cons_se += 1
else:
break
# Including the move, we need to have three other consecutive pieces to be a winner
result.consecutives.append(cons_left + cons_right)
if cons_left + cons_right >= 3:
log.info(
"Winner found at col {0} and row {1} for player {2} going left ({3}) and right ({4})"
.format(move_col, move_row, move_mark, cons_left, cons_right))
result.winner = True
result.consecutives.append(cons_up + cons_down)
if cons_up + cons_down >= 3:
log.info(
"Winner found at col {0} and row {1} for player {2} going up ({3}) and down ({4})"
.format(move_col, move_row, move_mark, cons_up, cons_down))
result.winner = True
result.consecutives.append(cons_nw + cons_se)
if cons_nw + cons_se >= 3:
log.info(
"Winner found at col {0} and row {1} for player {2} going nw ({3}) and se ({4})"
.format(move_col, move_row, move_mark, cons_nw, cons_se))
result.winner = True
result.consecutives.append(cons_ne + cons_sw)
if cons_ne + cons_sw >= 3:
log.info(
"Winner found at col {0} and row {1} for player {2} going ne ({3}) and sw ({4})"
.format(move_col, move_row, move_mark, cons_ne, cons_sw))
result.winner = True
return result
def reload(*p):
bot.interact('reld')
vk.initSelf()
log.info('Reloaded!')
return 'Reloaded!'
def _parse_proposal_section(self, data):
assert self.website
for category_name, proposals_data in data.items():
category = district = None
if category_name in {"State"}:
district = District.objects.get(name="Michigan")
elif category_name in {"County"}:
district = self.precinct.county
elif category_name in {
"City",
"Township",
"Village",
"Authority",
"Authority (Custom Region)",
"Metropolitan",
}:
district = self.precinct.jurisdiction
elif category_name in {
"Community College",
"Intermediate School",
"Local School",
"District Library",
"Ward",
}:
category = DistrictCategory.objects.get(name=category_name)
else:
raise exceptions.UnhandledData(
f"Unhandled category {category_name!r} on {self.website.mvic_url}"
)
for proposal_data in proposals_data:
if category and category.name == "Ward":
district, created = District.objects.get_or_create(
category=category,
name=self.precinct.get_ward_label(
f"Ward {self.precinct.ward}"),
)
if created:
log.info(f"Created district: {district}")
if district is None:
assert category, f"Expected category: {proposals_data}"
possible_category_names = [category.name]
if category.name == "District Library":
possible_category_names.extend([
"Public Library",
"Community Library",
"Library District",
"Library",
])
elif category.name == "Community College":
possible_category_names.extend([
"College",
])
elif category.name == "Local School":
possible_category_names.extend([
"Public Schools",
"Area Schools",
"Area School District",
"Public School",
"Community Schools",
"Community School District",
"Community School",
"Consolidated Schools District",
"School District",
"Area School System",
"Schools",
])
elif category.name == "Intermediate School":
possible_category_names.extend([
"Regional Education Service Agency",
"Regional Educational Service Agency",
"Regional Education Service",
"Area Educational Service Agency",
"Educational Service",
])
original_exception = None
for category_name in possible_category_names:
try:
district_name = helpers.parse_district_from_proposal(
category_name,
proposal_data["text"],
self.website.mvic_url,
)
except ValueError as e:
if original_exception is None:
original_exception = e
else:
break
else:
raise original_exception # type: ignore
district, created = District.objects.get_or_create(
category=category, name=district_name)
if created:
log.info(f"Created district: {district}")
if proposal_data["text"] is None:
raise exceptions.MissingData(
f"Proposal text missing on {self.website.mvic_url}")
proposal, created = Proposal.objects.update_or_create(
election=self.election,
district=district,
name=proposal_data["title"],
defaults={"description": proposal_data["text"]},
)
if created:
log.info(f"Created proposal: {proposal}")
proposal.precincts.add(self.precinct)
proposal.save()
yield proposal
def _parse_nonpartisan_section(self, data):
assert self.website
for category_name, positions_data in data.items():
for position_data in positions_data:
category = district = None
if category_name in {
"City",
"Township",
"Village",
"Authority",
"Metropolitan",
}:
if position_data["district"]:
category = self.precinct.jurisdiction.category
else:
district = self.precinct.jurisdiction
elif category_name in {
"Community College",
"Local School",
"Intermediate School",
"Library",
}:
category = DistrictCategory.objects.get(name=category_name)
elif category_name in {"Judicial"}:
pass # district will be parsed based on position name
else:
raise exceptions.UnhandledData(
f"Unhandled category {category_name!r} on {self.website.mvic_url}"
)
position_name = position_data["name"]
if district is None:
if category is None:
if position_name in {"Justice of Supreme Court"}:
district = District.objects.get(name="Michigan")
elif position_name in {"Judge of Court of Appeals"}:
category = DistrictCategory.objects.get(
name="Court of Appeals")
elif position_name in {"Judge of Municipal Court"}:
category = DistrictCategory.objects.get(
name="Municipal Court")
elif position_name in {"Judge of Probate Court"}:
category = DistrictCategory.objects.get(
name="Probate Court")
elif position_name in {"Judge of Circuit Court"}:
category = DistrictCategory.objects.get(
name="Circuit Court")
elif position_name in {"Judge of District Court"}:
category = DistrictCategory.objects.get(
name="District Court")
else:
raise exceptions.UnhandledData(
f"Unhandled position {position_name!r} on {self.website.mvic_url}"
)
if position_data["district"]:
district, created = District.objects.get_or_create(
category=category, name=position_data["district"])
if created:
log.info(f"Created district: {district}")
elif district is None:
log.warn(
f"Ballot {self.website.mvic_url} missing district: {position_data}"
)
district = self.precinct.jurisdiction
elif position_name in {"Commissioner by Ward"}:
category = DistrictCategory.objects.get(name="Ward")
district, created = District.objects.get_or_create(
category=category,
name=self.precinct.get_ward_label(
position_data["district"]),
)
if created:
log.info(f"Created district: {district}")
position, created = Position.objects.get_or_create(
election=self.election,
district=district,
name=position_data["name"],
term=position_data["term"] or "",
seats=position_data["seats"] or 0,
)
if created:
log.info(f"Created position: {position}")
position.section = "Nonpartisan"
position.precincts.add(self.precinct)
position.save()
yield position
for candidate_data in position_data["candidates"]:
assert candidate_data["party"] is None
party = Party.objects.get(name="Nonpartisan")
candidate, created = Candidate.objects.update_or_create(
position=position,
name=candidate_data["name"],
defaults={
"party": party,
"reference_url": candidate_data["finance_link"],
},
)
if created:
log.info(f"Created candidate: {candidate}")
yield candidate
def _parse_partisan_section(self, data, section=""):
assert self.website
for category_name, positions_data in data.items():
for position_data in positions_data:
category = district = None
if category_name in {"Presidential", "State", "State Board"}:
district = District.objects.get(name="Michigan")
elif category_name in {"City", "Township"}:
district = self.precinct.jurisdiction
elif category_name in {
"Congressional",
"Legislative",
"County",
"Delegate",
}:
pass # district parsed based on position name
else:
raise exceptions.UnhandledData(
f"Unhandled category {category_name!r} on {self.website.mvic_url}"
)
position_name = position_data["name"]
if district is None:
if position_name in {"United States Senator"}:
district = District.objects.get(name="Michigan")
elif position_name in {"Representative in Congress"}:
category = DistrictCategory.objects.get(
name="US Congress")
district, created = District.objects.get_or_create(
category=category, name=position_data["district"])
if created:
log.info(f"Created district: {district}")
elif position_name in {"State Senator"}:
category = DistrictCategory.objects.get(
name="State Senate")
district, created = District.objects.get_or_create(
category=category, name=position_data["district"])
if created:
log.info(f"Created district: {district}")
elif position_name in {
"Representative in State Legislature"
}:
category = DistrictCategory.objects.get(
name="State House")
district, created = District.objects.get_or_create(
category=category, name=position_data["district"])
if created:
log.info(f"Created district: {district}")
elif position_name in {"County Commissioner"}:
category = DistrictCategory.objects.get(
name=position_name)
district, created = District.objects.get_or_create(
category=category,
name=self.precinct.get_county_district_label(
position_data["district"]),
)
if created:
log.info(f"Created district: {district}")
elif position_name in {"Delegate to County Convention"}:
category = DistrictCategory.objects.get(
name="Precinct")
district, created = District.objects.get_or_create(
category=category,
name=self.precinct.get_precinct_label(),
)
if created:
log.info(f"Created district: {district}")
elif category_name in {"County"}:
district = self.precinct.county
else:
raise exceptions.UnhandledData(
f"Unhandled position {position_name!r} on {self.website.mvic_url}"
)
default_term = constants.TERMS.get(position_data["name"], "")
position, created = Position.objects.get_or_create(
election=self.election,
district=district,
name=position_data["name"],
term=position_data["term"] or default_term,
seats=position_data["seats"],
section=section,
)
if created:
log.info(f"Created position: {position}")
position.precincts.add(self.precinct)
position.save()
yield position
for candidate_data in position_data["candidates"]:
candidate_name = candidate_data["name"]
if candidate_name in {"Uncommitted"}:
log.debug(
f"Skipped placeholder candidate: {candidate_name}")
continue
if candidate_data["party"] is None:
raise exceptions.UnhandledData(
f"Expected party for {candidate_name!r} on {self.website.mvic_url}"
)
party = Party.objects.get(name=candidate_data["party"])
candidate, created = Candidate.objects.update_or_create(
position=position,
name=candidate_name,
defaults={
"party": party,
"reference_url": candidate_data["finance_link"],
},
)
if created:
log.info(f"Created candidate: {candidate}")
yield candidate
def _get_ballot(election: Election, precinct: Precinct) -> Ballot:
ballot, created = Ballot.objects.get_or_create(election=election,
precinct=precinct)
if created:
log.info(f"Created ballot: {ballot}")
return ballot
def onAuthSuccess(self, username):
info("Connected WA client (%s)" %username)
self.connected = True
self.methodsInterface.call("ready")
def _onexit(*p):
log.info('Received SIGTERM')
loop_thread.join(60)
vk.waitAllThreads()
log.info('Bye')
exit(0)
import log
import random
# 447,856 words
log.info("Loading dictionary...")
dictionary_file = open('dictionary.txt', 'r')
dictionary = dictionary_file.read().split('\n')
log.good("Loaded!")
def gen_credentials():
username = f'{dictionary[random.randint(0, 447856)]}{random.randint(100, 999)}'
password = f'{dictionary[random.randint(0, 447856)]}{random.randint(100000, 999999)}'
date_of_birth = (random.randint(1, 12), random.randint(1, 28),
random.randint(1940, 2002))
email = f'{username}@gmail.com'
log.good(
f"Generated username ({username}), password ({password}), email ({email}), and date of birth ({str(date_of_birth)})"
)
return (username, password, date_of_birth, email)
def cleanup(self):
if self._com is not None:
log.info("Closing connection to: %s", self.dev)
self._com.close()
def save_to_stream(output_stream):
mod = CFG_pb2.Module()
num_subs, num_blocks, num_insts = 0, 0, 0
exclude_blocks = set()
for sub in program.subroutines():
num_subs += 1
if program.Subroutine.VISIBILITY_IMPORTED == sub.visibility:
exclude_blocks.update(sub.blocks)
log.debug("Not serializing {} blocks".format(len(exclude_blocks)))
referenced_blocks = set()
addressed_blocks = set()
for block in program.basic_blocks():
if block in exclude_blocks:
continue
if not len(block.instructions):
referenced_blocks.add(block)
log.error("Block {:08x} has no instructions.".format(block.ea))
continue
num_blocks += 1
log.info("Serializing block {:08x}.".format(block.ea))
b = mod.blocks.add()
b.address = block.ea
if block.address_is_taken:
addressed_blocks.add(block)
for inst in block:
i = b.instructions.add()
i.address = inst.ea
i.bytes = inst.bytes
num_insts += 1
for block in addressed_blocks:
mod.addressed_blocks.append(block.ea)
for block in referenced_blocks:
mod.referenced_blocks.append(block.ea)
for sub in program.subroutines():
if not sub.name:
continue
if program.Subroutine.VISIBILITY_INTERNAL == sub.visibility:
continue
nb = mod.named_blocks.add()
nb.name = sub.name
nb.address = sub.ea
if program.Subroutine.VISIBILITY_IMPORTED == sub.visibility:
nb.visibility = CFG_pb2.IMPORTED
elif program.Subroutine.VISIBILITY_EXPORTED == sub.visibility:
nb.visibility = CFG_pb2.EXPORTED
log.info("Serializing {} subroutines".format(num_subs))
log.info("Serializing {} blocks".format(num_blocks))
log.info("Serializing {} instructions".format(num_insts))
output_stream.write(mod.SerializeToString())
def handle_nonpartisan_section(table: element.Tag, *, election: Election,
precinct: Precinct, **_) -> Optional[Proposal]:
td = table.find(class_='section')
if td and td.text != "NONPARTISAN SECTION":
return None
if table.find(class_='proposalTitle'):
return None
# Set party
party = Party.objects.get(name="Nonpartisan")
# Parse category
category = None
td = table.find(class_='division')
if not category and td:
division = helpers.titleize(td.text)
if division in {"Judicial"}:
pass # parse category from 'office'
else:
log.debug(f'Parsing category from division: {td.text!r}')
category = DistrictCategory.objects.get(
name=helpers.clean_district_category(division))
td = table.find(class_='mobileOnly')
if not category and td:
mobileonly = helpers.titleize(td.text)
if mobileonly not in {"Judicial"}:
log.debug(f'Parsing category from mobileOnly: {td.text!r}')
category = DistrictCategory.objects.get(
name=helpers.clean_district_category(mobileonly))
td = table.find(class_='office')
if not category and td:
office = helpers.titleize(td.text)
log.debug(f'Parsing category from office: {td.text!r}')
if office in {"Justice of Supreme Court"}:
category = DistrictCategory.objects.get(name="State")
else:
category = DistrictCategory.objects.get(
name=helpers.clean_district_category(office))
log.info(f'Parsed {category!r}')
assert category
# Parse district
district = None
td = table.find(class_='term')
if td:
if category.name == "State":
log.debug(f'Assuming district is state from {category}')
district = District.objects.get(category=category, name="Michigan")
elif category.name in {
"City", "Township", "Metropolitan", "Authority"
}:
log.debug(f'Assuming district is jurisdiction from {category}')
district = precinct.jurisdiction
elif category.name in {"Library"} and 'vote for' in td.text.lower():
log.warn(f'Assuming district is jurisdiction from {category}')
district = precinct.jurisdiction
else:
log.debug(f'Parsing district from term: {td.text!r}')
assert 'term' not in td.text.lower()
assert 'vote for' not in td.text.lower()
district_name = helpers.titleize(td.text)
district, created = District.objects.get_or_create(
category=category, name=district_name)
if created:
log.warn(f'Added missing district: {district}')
log.info(f'Parsed {district!r}')
assert district
# Parse position
office = table.find(class_='office').text
terms = table.find_all(class_='term')
term = ''
if len(terms) >= 2:
if "term" in terms[-2].text.lower():
term = terms[-2].text
if "position" in terms[1].text.lower(
) or "judgeship" in terms[1].text.lower():
assert term, f'Expected term: {term!r}'
term += f', {terms[1].text}'
seats = terms[-1].text
else:
seats = terms[-1].text
log.debug(f'Parsing position from: {office!r} for {term!r} when {seats!r}')
assert "vote for" in seats.lower()
position, _ = Position.objects.get_or_create(
election=election,
district=district,
name=helpers.titleize(office),
term=term,
seats=int(seats.strip().split()[-1]),
)
log.info(f'Parsed {position!r}')
assert position
# Add precinct
position.precincts.add(precinct)
position.save()
# Parse candidates
for td in table.find_all(class_='candidate'):
log.debug(f'Parsing candidate: {td.text!r}')
candidate_name = td.text.strip()
if candidate_name == "No candidates on ballot":
log.warn(f'No {party} candidates for {position}')
break
candidate, _ = Candidate.objects.get_or_create(name=candidate_name,
party=party,
position=position)
log.info(f'Parsed {candidate!r}')
return position
action='store',
type='string',
help='ip and port of server')
parser.add_option('-f',
'--fps',
dest='fps',
default=30,
action='store',
type='int',
help='frames per second')
options, args = parser.parse_args()
if options.layout == 'supported_files/ledwall15x9.json':
log.info(
file, "\nNo layout selected, using default layout: " +
str(options.layout) + "\n")
if not options.layout:
parser.print_help()
print()
log.warning(file, 'ERROR: you must specify a layout file using --layout')
print()
sys.exit(1)
#-------------------------------------------------------------------------------
# parse layout file
print
print
log.header(file, ' parsing FC layout file')
def handle_partisan_section(
table: element.Tag,
*,
election: Election,
precinct: Precinct,
party: Optional[Party],
**_,
) -> Optional[Position]:
if party and party.name == "Nonpartisan":
return None
if table.get('class') != ['tblOffice']:
return None
td = table.find(class_='section')
if td and td.text == "NONPARTISAN SECTION":
return None
# Parse category
category = None
td = table.find(class_='division')
if td:
category_name = helpers.titleize(td.text)
if category_name in {"State Board"}:
log.debug(f'Assuming category from division: {category_name}')
category = DistrictCategory.objects.get(name="State")
elif category_name not in {"Congressional", "Legislative", "Delegate"}:
log.debug(f'Parsing category from division: {td.text!r}')
category = DistrictCategory.objects.get(name=category_name)
if not category:
td = table.find(class_='office')
if td:
office = helpers.titleize(td.text)
if office == "United States Senator":
log.debug(f'Parsing category from office: {td.text!r}')
category = DistrictCategory.objects.get(name="State")
elif office == "Representative In Congress":
log.debug(f'Parsing category from office: {td.text!r}')
category = DistrictCategory.objects.get(name="US Congress")
elif office == "State Senator":
log.debug(f'Parsing category from office: {td.text!r}')
category = DistrictCategory.objects.get(name="State Senate")
elif office == "Representative In State Legislature":
log.debug(f'Parsing category from office: {td.text!r}')
category = DistrictCategory.objects.get(name="State House")
if not category:
class_ = 'mobileOnly'
td = table.find(class_=class_)
if td:
category_name = helpers.titleize(td.text)
log.debug(f'Parsing category from {class_!r}: {td.text!r}')
if category_name in {"State Board"}:
category = DistrictCategory.objects.get(name="State")
else:
category = DistrictCategory.objects.get(name=category_name)
log.info(f'Parsed {category!r}')
assert category
# Parse district
district = None
td = table.find(class_='office')
if td:
office = helpers.titleize(td.text)
if category.name == "State":
log.debug(f'Assuming state position: {office}')
district = District.objects.get(category=category, name="Michigan")
elif office in {"Governor", "Governor and Lieutenant Governor"}:
# TODO: Delete?
log.debug(f'Parsing district from office: {td.text!r}')
district = District.objects.get(category=category, name="Michigan")
elif office == "United States Senator":
# TODO: Delete?
log.debug(f'Parsing district from office: {td.text!r}')
district = District.objects.get(category=category, name="Michigan")
elif category.name == "County":
log.debug(f'Parsing district from office: {td.text!r}')
district = precinct.county
elif category.name in {"City", "Township"}:
log.debug(f'Assuming jurisdiction position: {office}')
district = precinct.jurisdiction
elif category.name == "Precinct":
log.debug(f'Parsing district from office: {td.text!r}')
district = precinct
else:
td = table.find(class_='term')
log.debug(f'Parsing district from term: {td.text!r}')
assert 'term' not in td.text.lower()
assert 'vote for' not in td.text.lower()
district_name = helpers.titleize(td.text)
district, created = District.objects.get_or_create(
category=category, name=district_name)
if created:
log.warn(f'Added missing district: {district}')
log.info(f'Parsed {district!r}')
assert district
# Parse position
office = table.find(class_='office').text
terms = table.find_all(class_='term')
if len(terms) >= 3:
term = terms[-2].text
assert "Term" in term
else:
term = ''
seats = terms[-1].text
log.debug(f'Parsing position from: {office!r} when {seats!r}')
assert 'term' not in office.lower()
assert 'vote for' not in office.lower()
position_name = helpers.titleize(office)
seats = int(seats.strip().split()[-1])
if isinstance(district, Precinct):
position_name = f'{position_name} ({party} | {district})'
district = None
position, _ = Position.objects.get_or_create(election=election,
district=district,
name=position_name,
term=term,
seats=seats)
log.info(f'Parsed {position!r}')
if position.seats != seats:
bugsnag.notify(f'Number of seats for {position} differs: '
f'{position.seats} vs. {seats}')
# Add precinct
position.precincts.add(precinct)
position.save()
# Parse parties
parties = []
for td in table.find_all(class_='party'):
log.debug(f'Parsing party: {td.text!r}')
party = Party.objects.get(name=td.text.strip())
log.info(f'Parsed {party!r}')
parties.append(party)
log.debug(f'Expecting {len(parties)} candidate(s) for {position}')
# Parse candidates
has_running_mates = False
for index, td in enumerate(table.find_all(class_='candidate')):
log.debug(f'Parsing candidate: {td.text!r}')
candidate_name = td.text.strip()
if candidate_name == "No candidates on ballot":
log.warn(f'No {party} candidates for {position}')
break
if " and " in position.name and index % 2:
log.warn(f'Skipped running mate: {candidate_name}')
has_running_mates = True
continue
party = parties[index // (2 if has_running_mates else 1)]
candidate, _ = Candidate.objects.get_or_create(name=candidate_name,
party=party,
position=position)
log.info(f'Parsed {candidate!r}')
return position
def killSwitch():
end = "run"
global run_audio
global run_main
global run_HTTPserver
run_audio = False
#sudo kill $(ps aux | grep 'fadecandy' | awk '{print $2}')
#sudo kill $(ps aux | grep 'main.py' | awk '{print $2}')
log.info(file, "killing HTTPserver")
log.info(file, "Sending one last request to kill the HTTPserver")
HTTPserver.run = False
try: # try to send one last request to kill the server
r = requests.get("http://localhost:321")
except:
pass
time.sleep(.5)
log.info(file, "killing googleAssistant")
googleAssistant.run = False
print()
print()
log.info(file, "killing fadecandy server")
os.system("sudo kill $(ps aux | grep 'fadecandy' | awk '{print $2}')")
time.sleep(.5)
log.info(file, "Killing main")
run_main = False
log.info(file, "Setting pixels to 0,0,0")
pixels = [(0, 0, 0) for ii, coord in enumerate(coordinates)
] #set all the pixels to off
client.put_pixels(pixels, channel=0)
print()
print()
log.info(
file,
"Is the HTTPserver thread running " + str(t_HTTPserver.is_alive()))
log.info(
file, "Is the googleAssistant thread running " +
str(t_googleAssistant.is_alive()))
print()
return ()
def info(msg):
log.info(msg,"registry")
def getVelocities(self, numRobot, myVelocities, beaconCoord, robotCoord,
alienCoord, now):
#print "beacon coord", beaconCoord.getVelocities()
#print "robot coord", robotCoord.getVelocities()
#print "alien coord", alienCoord.getVelocities()
self.setTime(now.secs + now.nsecs / (10**9)) # segundos.milisegundos
#print ("Tempo decorrido %4.9f" %(self.actualTime - self.previousTime))
#atualiza a velocidade real
self.realLinearVelocity, self.realAngularVelocity = myVelocities.getVelocities(
)
#atualiza a existencia dos objetos
self.updateDetectedObjects(beaconCoord, robotCoord, alienCoord)
#age conforme os objetos detectados
if (self.hasBeacon and self.hasRobot): #Escorting
self.status = 3
#--- bloco comentado --- 2016.02.05
#self.circularCoefToBeacon = self.getCircularCoefToBeacon()
self.propAngularCoefToRobot = self.getPropAngularCoefToRobot()
self.proximityCoefToRobot = self.getProximityCoefToRobot(
) # * self.getPropAngularCoefToRobot()
#self.linearCoefBeaconToRobot = self.getLinearCoefBeaconToRobot(beaconCoord, robotCoord)
#-comentado anteriormente-#self.linearVelocity = sp.linear_velocity + sp.linear_velocity * self.linearCoefBeaconToRobot * self.proximityCoefToRobot
self.linearVelocity = sp.linear_velocity + sp.linear_velocity * self.propAngularCoefToRobot * self.proximityCoefToRobot
##mantenha o raio desejado
#self.angularVelocity = self.linearVelocity / sp.desired_radius + self.circularCoefToBeacon
#---fim do bloco comentado ---
elif (self.hasBeacon and not self.hasRobot): #Circulating
self.status = 2
circRadiusCoef, appRadiusCoef = self.getTransitionCoefs()
kpA = self.getPropAngularCoefToBeacon()
error_radius = sp.desired_radius - self.beacon.linear
appRadius = -(self.beacon.x**2 + self.beacon.y**2 -
sp.desired_radius**2) / (
2 * (sp.desired_radius - self.beacon.y))
circRadius = sp.desired_radius + error_radius
radius = circRadiusCoef * circRadius + appRadiusCoef * appRadius
ang_vel = self.linearVelocity / radius
hl_ang_vel = self.getHardLimited(
sp.min_angular_velocity, ang_vel,
sp.max_angular_velocity) + kpA * sp.delta_angular
self.angularVelocity = self.getHardLimited(sp.min_angular_velocity,
hl_ang_vel,
sp.max_angular_velocity)
log.info(str(now))
print "kpA", kpA, "error_radius", error_radius
print "appRadius", appRadius, "appRadiusCoef", appRadiusCoef, "=", appRadius * appRadiusCoef
print "circRadius", circRadius, "circRadiusCoef", circRadiusCoef, "=", circRadius * circRadiusCoef
print "radius", radius, "ang_vel", ang_vel, "hl_ang_vel", hl_ang_vel, "angularVelocity", self.angularVelocity
self.approachRadius = appRadius
self.beacon.prn()
elif (not self.hasBeacon and self.hasRobot): #Avoiding
self.status = 1
#self.proximityCoefToRobot = self.getProximityCoefToRobot()
#self.propAngularCoefToRobot = self.getPropAngularCoefToRobot()
#self.propDistCoefToRobot = self.getPropDistCoefToRobot()
#self.linearVelocity = sp.linear_velocity + sp.linear_velocity * self.proximityCoefToRobot * self.propAngularCoefToRobot
#self.angularVelocity = self.linearVelocity / self.obtainedDistanceToRobot + sp.angular_velocity * self.proximityCoefToRobot * self.propDistCoefToRobot
else: #status == "Seeking"
self.status = 0
self.linearVelocity = sp.linear_velocity
#self.angularVelocity = sp.angular_velocity + sp.angular_velocity * np.random.random() * 0.1
decimal_random_rate = np.random.random() / 10.0
self.angularVelocity = sp.angular_velocity * (1 +
decimal_random_rate)
### delimiter ###
self.linearVelocity = self.getDelimitedLinearVelocity(
self.linearVelocity)
#self.angularVelocity = self.getDelimitedAngularVelocity(self.angularVelocity)
### Parada de Seguranca ###
self.linearVelocity = self.verifyMinDistances(numRobot,
self.linearVelocity)
###------------------------------###
### Atualizacao das propriedades ###
myVelocities.angular = self.angularVelocity
myVelocities.linear = self.linearVelocity
#atualiza rotacoes e velocidades tangenciais #monitoramento
self.rightLinearVelocity = myVelocities.angular + myVelocities.linear
self.leftLinearVelocity = -2 * myVelocities.linear - myVelocities.angular
self.rightWheelRotation = self.rightLinearVelocity / (
sp.wheel_diameter * np.pi)
self.leftWheelRotation = self.leftLinearVelocity / (sp.wheel_diameter *
np.pi)
#self.printAll(numRobot)
return myVelocities
screen_size = [700, 700]
xy_center = [screen_size[0] / 2, screen_size[1] / 2]
width = 800
height = 600
centerX = width / 2
centerY = height / 2
# 3D to 2D projection parameters
fov = 256
viewer_distance = 2.2
inv = math.pi / 2
print()
log.info('Startup...')
# print("Arguments parsing if needed...")
argsparser = argparse.ArgumentParser(description="Aurora for LJ")
argsparser.add_argument(
"-r",
"--redisIP",
help="IP of the Redis server used by LJ (127.0.0.1 by default) ",
type=str)
argsparser.add_argument("-m",
"--myIP",
help="IP to bind (0.0.0.0 by default) ",
type=str)
argsparser.add_argument("-s",
"--scene",
help="LJ scene number (0 by default)",
type=int)
def dash_connection():
log.info('Dashboard connected')
def send(self, target, text):
self.wait_connected()
self.methodsInterface.call("message_send", (target, text.encode("utf-8")))
info((" >>> WA %s: %s" %(target, text)).encode("utf-8"))
def classify_req(sock):
global last_frame_time, frames_received, positives_last_frame, collecting_negs
start_time = time.time()
log.info("got connection")
# read and decode the header
read_start = time.time()
hdr_fmt = 'ccccIII'
hdr_buf = sock.recv(struct.calcsize(hdr_fmt))
m0, m1, m2, m3, w, h, d = struct.unpack(hdr_fmt, hdr_buf)
# make sure this message starts with the expected magic
if (str(m0 + m1 + m2 + m3, 'utf8')) != 'POOP':
sock.close()
return
# the request is good, read the frame
frame = b''
while True:
chunk = sock.recv(w * h * d - len(frame))
if len(chunk) == 0 and len(frame) > 0:
break
frame += chunk
img = Image.frombuffer('RGB', (w, h), frame)
log.info('Frame read took %f sec', time.time() - read_start)
is_poop = False
frames_received += 1
# try:
# do classification here
collecting_pos = has_cli_arg('collect') and has_cli_arg('positives')
if collecting_negs:
ds.store(0).tile(img, tiles=10, size=(16, 16))
elif collecting_pos:
ds.store('src')._store(img)
else:
classifications, visualization = classifier.classify(img)
positives = classifications.sum()
is_poop = positives + positives_last_frame >= 4
positives_last_frame = positives
if is_poop:
# the frame was classified as poop. Save it to confirm later
DataStore('/tmp/pood/ds').store(1).image(img)
visualization = np.dstack((visualization, np.ones(visualization.shape[0:2], dtype='uint8') * 255))
transmit_start = time.time()
socketio.emit('size', {'w': w, 'h': h})
socketio.emit('frame', {'data': visualization.flatten().tobytes()})
log.info('Transmit frame took %f sec', time.time() - start_time)
# with open('/tmp/pood.classification.png', 'wb') as fp:
# Image.fromarray(visualization, mode="RGB").save(fp)
if not is_poop and np.random.rand() < 0.01:
ds.store(0).tile(img, tiles=10, size=(16, 16))
last_frame_time = time.time()
# except:
# pass
# send result back
sent = sock.sendall(struct.pack('I', int(not is_poop)))
log.info('Classification request took %f sec', time.time() - start_time)
def onAuthFailed(self, username, reason):
info("Auth Failed: %s" %reason)
self.connected = False
def fetch_registration_status(
self,
*,
track_missing_data: bool = "staging" not in settings.BASE_URL
) -> RegistrationStatus:
data = helpers.fetch_registration_status_data(self)
if not data["registered"]:
return RegistrationStatus(registered=False)
districts: List[District] = []
county = jurisdiction = None
for category_name, district_name in sorted(data["districts"].items()):
if not district_name:
log.debug(f"Skipped blank district: {category_name}")
continue
if category_name in ["Ward", "Precinct"]:
log.debug(f"Skipped category: {category_name}")
continue
category = DistrictCategory.objects.get(name=category_name)
if category.name == "County":
district_name = district_name.replace(" County", "")
district, created = District.objects.get_or_create(
category=category, name=district_name)
if created:
message = f"Created district: {district}"
log.info(message)
districts.append(district)
if district.category.name == "County":
county = district
if district.category.name == "Jurisdiction":
jurisdiction = district
if data["districts"]["Ward"]:
precinct, created = Precinct.objects.get_or_create(
county=county,
jurisdiction=jurisdiction,
ward=data["districts"]["Ward"],
number=data["districts"]["Precinct"],
)
else:
precinct, created = Precinct.objects.get_or_create(
county=county,
jurisdiction=jurisdiction,
number=data["districts"]["Precinct"],
)
if created:
message = f"Created precinct: {precinct}"
log.info(message)
if track_missing_data:
bugsnag.notify(
exceptions.MissingData(message),
metadata={
"voter": {
"first_name": self.first_name,
"last_name": self.last_name,
"birth_data": self.birth_date,
"zip_code": self.zip_code,
}
},
)
status = RegistrationStatus(
registered=data["registered"],
ballot=data["ballot"],
absentee=data["absentee"],
absentee_application_received=data["absentee_dates"]
["Application Received"],
absentee_ballot_sent=data["absentee_dates"]["Ballot Sent"],
absentee_ballot_received=data["absentee_dates"]["Ballot Received"],
polling_location=list(data["polling_location"].values()),
dropbox_locations=data["dropbox_locations"],
recently_moved=data["recently_moved"],
precinct=precinct,
)
status.districts = districts
return status
