def show_pattern_and_template(dna):
from executor import Executor
e = Executor(dna)
e.explicit_rna_items = True
pattern = e.pattern()
pattern.append('EoP')
template = e.template()
template.append('EoT')
e.item_starts.append(e.parser.index)
s1 = []
s2 = []
for item, begin, end in zip(
pattern+template,
e.item_starts,
e.item_starts[1:]):
e1 = ''.join(e.dna[begin:end])
e2 = str(item)
if len(e1) > len(e2):
e2 += ' '*(len(e1)-len(e2))
else:
e1 += ' '*(len(e2)-len(e1))
s1.append(e1)
s2.append(e2)
print ' '.join(s1)
print ' '.join(s2)
class REPL(Cmd):
prompt = 'repl:>> '
def __init__(self):
Cmd.__init__(self)
self.ex = Executor()
self.trans = translator.Translator()
def default(self, line):
try:
y = yaml.load(line)
print 'yaml:', y
js = self.trans.translate(y)
print 'generated js:', js
print self.ex.execute(js)
except Exception as e:
print e
def do_EOF(self, line):
return True
def do_quit(self, line):
return True
def do_reload(self, line):
reload(translator)
self.trans = translator.Translator()
return False
def _process_event(self, event):
global config
if event:
# This still needs some work
# dependencies = self._check_for_dependencies(config, event)
dependencies = []
self.job_count += 1
job_id = "%s:%d" % (self.name, self.job_count)
try:
# Launch test runs
tests = []
for test in config['tests']:
testexec = Executor(test, event, dependencies)
tests.append(testexec)
testexec.start()
logging.info('Started %s test execution...',
test['test-name'])
# Wait for all executions to finish
for test in tests:
test.join()
logging.info('All tests completed')
# Report the results
self._publish_results_to_gerrit(
config, tests, job_id)
# Throttle things a little bit
time.sleep(5)
except Exception:
logging.exception('Error in event processing!')
class LiquipyDatabase(object):
"""
Main interface for Liquipy
"""
def __init__(self, host=DEFAULT['host'],
database=DEFAULT['database'],
username=DEFAULT['username'],
password=DEFAULT['password'],
tempDir=DEFAULT['tempDir']):
self.liquibaseExecutor = LiquibaseExecutor(host, database, username, password)
self.tempDir = tempDir
self.outputXmlChangeLogFilePath = self.tempDir + "/liquipy_changelog.xml"
def initialize(self, yamlPath):
rawYaml = open(yamlPath, 'r').read()
try:
changes = yaml.load(rawYaml)
except yaml.scanner.ScannerError as e:
msg = "Error parsing input YAML file '%s':\n%s" % (yamlPath, e)
raise Exception(msg)
changeSetWriter = ChangeSetWriter(self.outputXmlChangeLogFilePath)
changeSetWriter.write(changes)
def update(self):
self.liquibaseExecutor.run(self.outputXmlChangeLogFilePath, 'update')
def kill_device(device):
cmd = "ps xww | grep Simulator.app | grep -s {0} | grep -v grep | awk '{{print $1}}'".format(device.uuid)
output = Executor.execute(cmd)
if output == '':
return
pid = int(output)
if pid > 0:
Executor.execute('kill {0}'.format(pid))
class Hostapd(object):
""" Wrapper class for UCI (OpenWRT) commands """
executor = None
devicename = None
def __init__(self, config = "/etc/hostapd/hostapd.conf", wifi_restart_command, executor=None):
if executor == None:
self.executor = Executor()
else:
self.executor = executor
self.config = config
self.restart_command = wifi_restart_command
def get_wifi_interface(self):
"Return the wifi interface name (e.g. 'wlan0')"
ret = self.executor.execute_cmd(['grep', '^interface', self.config, '|', 'cut','-f2','-d"="'])
if ret is None:
print 'No WiFi device name found.'
return ret
def get_bridge_interface(self):
"Return the bridge interface name (e.g. 'br0')"
ret = self.executor.execute_cmd(['grep', '^bridge', self.config, '|', 'cut','-f2','-d"="'])
if ret is None:
print 'No bridge device name found.'
return ret
def get_wifi_mode(self):
"Get operation mode (e.g. n)"
ret = self.executor.execute_cmd(['grep', '^hw_mode', self.config, '|', 'cut','-f2','-d"="'])
if ret is None:
print 'No mode found in config.'
return ret
def set_channel(self, channel):
"Sets the wifi channel. Requires commit and restart of WiFi for changes to take effect"
self.executor.execute_cmd(['sudo','cat',self.config,'|','sed','-e',"s/channel=[0-9][0-9]*/channel=%d/g" % channel, '>', '/tmp/tmp_hostapd.conf'])
self.executor.execute_cmd(['sudo','mv','/tmp/tmp_hostapd.conf',self.config])
def restart():
"Restart hostapd"
self.executor.execute_cmd(wifi_restart_command)
def get_wifi_ssid(self):
"Return the wifi ssid (e.g. for 'node1-wifi')"
ret = self.executor.execute_cmd(['grep', '^ssid', self.config, '|', 'cut','-f2','-d"="'])
if ret is None:
print 'No SSID found in config.'
return ret
class LiquipyDatabase(object):
"""
Main interface for Liquipy
"""
def __init__(
self,
host=DEFAULT["host"],
database=DEFAULT["database"],
username=DEFAULT["username"],
password=DEFAULT["password"],
tempDir=DEFAULT["tempDir"],
):
self.liquibaseExecutor = LiquibaseExecutor(host, database, username, password)
self.tempDir = tempDir
self.outputXmlChangeLogFilePath = self.tempDir + "/liquipy_changelog.xml"
def initialize(self, yamlPath):
self.changes = self.inputYamlToChangeSets(yamlPath)
changeSetWriter = ChangeSetWriter(self.outputXmlChangeLogFilePath)
changeSetWriter.write(self.changes)
def inputYamlToChangeSets(self, yamlPath):
rawYaml = open(yamlPath, "r").read()
try:
changes = yaml.load(rawYaml)
except yaml.scanner.ScannerError as e:
msg = "Error parsing input YAML file '%s':\n%s" % (yamlPath, e)
raise Exception(msg)
if "include" in changes.keys():
relativeTargetDir = changes["include"]["directory"]
currentDir = join(split(yamlPath)[:-1])[0]
targetDir = join(currentDir, relativeTargetDir)
try:
dirFiles = listdir(targetDir)
except Exception:
raise Exception('Included directory "' + targetDir + '" does not exist')
migrationFiles = [join(targetDir, f) for f in dirFiles if f.endswith(".yml")]
for includedMigration in migrationFiles:
includeChanges = self.inputYamlToChangeSets(includedMigration)
changes.update(includeChanges)
del changes["include"]
return changes
def update(self):
print "Running all migrations..."
self.liquibaseExecutor.run(self.outputXmlChangeLogFilePath, "update")
def rollback(self, tagName):
print "Rolling back to %s..." % (tagName,)
self.liquibaseExecutor.run(self.outputXmlChangeLogFilePath, "rollback", tagName)
def getTags(self):
return [
{"tag": self.changes[c]["tag"], "changeSet": c} for c in self.changes.keys() if "tag" in self.changes[c]
]
def delete(self):
kw = dict((self._basic_fields[i], self.values[i]) for i in self._pk)
kw['multi'] = None
kw['alias'] = self.__alias__
kw['uin'] = self._uin
w = ' AND '.join('%s = :%s' % (self._basic_fields[i], i) for i in self._pk)
if w:
w = 'WHERE ' + w
Executor.save('DELETE FROM %s %s' % (self.__table__, w), **kw)
def save(self, **kw):
isnew = kw.pop('isnew', any(self.values[i] is None for i in self._pk))
if self._dirty or isnew:
# set default volumes
for i in self._basic_fields.keys():
if self.values[i] is None:
x = self._fields[i]._default
if isinstance(x, (FunctionType, MethodType, BuiltinFunctionType, BuiltinMethodType,)):
x = x()
elif isinstance(x, Sequence):
# print 'W1', x
x = x()
# print 'W2', x
self.__setattr__(i, x)
kw = {'alias': self.__alias__}
if isnew:
fields = []
for i in self._dirty:
k = self._basic_fields[i]
kw[k] = self.values[i]
fields.append(k)
sql = 'INSERT INTO %s (%s) VALUES (%s)' % (self.__table__,
','.join(fields),
','.join(' :%s' % i for i in fields))
else:
sets = []
whr = []
for i in self._dirty:
k = self._basic_fields[i]
kw[k] = self.values[i]
sets.append('%s = :%s' % (k, k))
for pk in self._pk:
k = self._basic_fields[pk]
kw[k] = self.old_values[pk]
whr.append('%s = :%s' % (k, k))
sql = 'UPDATE %s SET %s WHERE %s' % (self.__table__,
', '.join(sets),
' AND '.join(whr))
self._dirty = []
kw['uin'] = self._uin
Executor.save(sql, **kw)
def main():
current_os = sys.platform
os_commands = {'linux': 'xdg-open', 'win32': 'start', 'darwin': 'open'}
script_abs_path = os.path.dirname(os.path.abspath(__file__))
script_parent_dir = script_abs_path.rsplit(os.sep, 1)[0]
sys.path.append(script_parent_dir)
# Check if settings directory exists and create it if it doesn't
settings_dir = '{}{}settings'.format(script_abs_path, os.sep)
if not os.path.exists(settings_dir):
Executor.execute(["mkdir", settings_dir])
# Check if sync_list file exists and create it if it doesn't
sync_list_path = '{}{}sync_list.txt'.format(settings_dir, os.sep)
if not os.path.exists(sync_list_path):
Executor.execute(["touch", sync_list_path])
# Get needed command to open default text editor depending on the OS
command = None
if 'linux' in current_os:
command = os_commands['linux']
elif 'win32' in current_os:
command = os_commands['win']
elif 'darwin' in current_os:
command = os_commands['darwin']
error_message = \
"""ERROR: An error occured while trying to open
"{}" for writing.
REASON: One possible reason is that your
operating system is not supported.
Your current operating system: {}
Supported operating systems: {}
If your operating system is not in the list or
it is but you still see this error
please give a feedback to support the development
of this app and you to be able to use it.
SOLUTION: For now you can edit the sync list manually
as you open it with some text editor."""\
.format(sync_list_path, current_os, ', '.join(os_commands.keys()))
if command is None or not Executor.execute([command, sync_list_path]):
print(error_message)
def __call__(self, parent):
if isinstance(self._cls, basestring):
self._cls = Executor.classes[self._cls]
# codtition
if self._condition:
c1, c2 = [i.strip() for i in self._condition.split('=')]
if c2.startswith(parent.__class__.__name__ + '.'):
c1, c2 = c2, c1
c1a, c1b = c1.split('.')
c2a, c2b = c2.split('.')
if parent.__class__.__name__ <> c1a or self._cls.__name__ <> c2a:
raise ReferenceException('Invalid condition "%s" by classes "%s" and "%s"' % (self._condition, c1, c2))
return self._cls._read(self._multi, **{c2b:parent.values[c1b]})
# if function of SQL generating
elif self._generator:
if isinstance(self._generator, basestring):
self._generator = parent.__class__.__dict__[self._generator]
sql, kw = self._generator(parent)
# None-классы т.е. SQL-функции
if self._cls is None:
return Executor.exec_sql(sql, multi=False, **kw).data[0]
# Внешние классы
else:
return self._cls._read(self._multi, sql=sql, **kw)
def __init__(self, config = "/etc/hostapd/hostapd.conf", wifi_restart_command, executor=None):
if executor == None:
self.executor = Executor()
else:
self.executor = executor
self.config = config
self.restart_command = wifi_restart_command
def __init__(self, limb, hover_distance = 0.15, verbose=True):
self._limb_name = limb # string
self._hover_distance = hover_distance # in meters
self._verbose = verbose # bool
self._executor = Executor(limb, verbose)
trajsvc = "baxter_adapt/imitation_server"
rospy.wait_for_service(trajsvc, 5.0)
self._trajsvc = rospy.ServiceProxy(trajsvc, Imitation)
def list_runtimes(self):
output = Executor.execute(CMD.format('runtimes'))
runtime_list = json.loads(output)['runtimes']
return [RunTime(runtime['availability'],
runtime['buildversion'],
runtime['identifier'],
runtime['name'],
runtime['version']) for runtime in runtime_list]
def __init__(self, host=DEFAULT['host'],
database=DEFAULT['database'],
username=DEFAULT['username'],
password=DEFAULT['password'],
tempDir=DEFAULT['tempDir']):
self.liquibaseExecutor = LiquibaseExecutor(host, database, username, password)
self.tempDir = tempDir
self.outputXmlChangeLogFilePath = self.tempDir + "/liquipy_changelog.xml"
def main():
config = Config()
executor = Executor(config)
if config.mode == 'train':
executor.train()
elif config.mode == 'sample':
executor.sample()
else:
executor.test(config.mode)
def __init__(self):
self.executor = Executor()
self.commands = {
'add':
self.executor.add,
'end':
self.executor.set_end_function,
'epsilon':
self.executor.set_epsilon,
'radian':
lambda radian: self.executor.set_radian(float(radian)),
'range':
lambda begging, end: self.executor.set_range(
float(begging), float(end)),
'del':
self.executor.remove,
'load':
self.load,
'custom':
self.executor.create_custom_function,
'maximize':
self.set_end_func_type,
'multithreading':
self.set_multi_threading_type,
'sizethreads':
lambda size: self.executor.set_number_of_threads(int(size)),
'execute':
self.executor.execute,
'calculate':
self.executor.calculate,
'clear':
self.executor.clear,
'clearall':
self.executor.clear_all,
'show':
self.show,
'help':
self.print_help,
'exit':
exit,
'#':
self.ignore,
'':
self.ignore,
}
def test_execute_sync_with_output(self, mocks):
self.mock_subprocess.Popen.return_value.communicate.return_value = 'opened'
result = Executor().execute_sync_with_output(['run', 'with', 'args'])
self.mock_subprocess.Popen.assert_called_with(['run', 'with', 'args'],
stdout='pipe',
stderr='pipe')
assert result == 'opened'
def test_files_created(self):
"""Test that we can run experiments and get output files"""
conf_name = 'myconf'
wl_name = 'mywl'
results_dir = os.path.join(self.te.LISA_HOME, 'results', self.res_dir,
'rtapp:{}:{}'.format(conf_name, wl_name))
if os.path.isdir(results_dir):
shutil.rmtree(results_dir)
experiments_conf = {
'confs': [{
'tag': conf_name
}],
"wloads": {
wl_name: {
"type": "rt-app",
"conf": {
"class": "profile",
"params": {
"mytask": {
"kind": "Periodic",
"params": {
"duty_cycle_pct": 10,
"duration_s": 0.2,
},
},
},
},
},
},
}
executor = Executor(self.te, experiments_conf)
executor.run()
self.assertTrue(
os.path.isdir(results_dir),
'Expected to find a directory at {}'.format(results_dir))
result_1_dir = os.path.join(results_dir, '1')
self.assertTrue(
os.path.isdir(result_1_dir),
'Expected to find a directory at {}'.format(result_1_dir))
def __init__(self, uri, exchangeName=None, routingKey=None, applicationId=None,
responseName=None, verbose=False,
monitorEnabled=True, monitorId=None, monitorInterval=None, monitorTimeout=None):
if logger.isEnabledFor(logging.DEBUG): logger.debug('Constructor begin ...')
self.__lock = threading.RLock()
self.__idle = threading.Condition(self.__lock)
self.__engine = Engine(**{
'uri': uri,
'exchangeName': exchangeName,
'exchangeType': 'direct',
'routingKey': routingKey,
'applicationId': applicationId,
'verbose': verbose
})
self.__executor = Executor(engine=self.__engine)
self.__tasks = {}
self.__timeoutHandler = None
self.__responseConsumer = None
if responseName is not None and type(responseName) is str:
self.__responseName = responseName
self.__executor.assertQueue(self.__responseName)
else:
self.__responseName = None
if monitorEnabled is not None and type(monitorEnabled) is bool:
self.__monitorEnabled = monitorEnabled
else:
self.__monitorEnabled = True
if monitorId is not None and type(monitorId) is str:
self.__monitorId = monitorId
else:
self.__monitorId = Util.getUUID()
if monitorInterval is not None and type(monitorInterval) is int:
self.__monitorInterval = monitorInterval
else:
self.__monitorInterval = 1
if monitorTimeout is not None and type(monitorTimeout) is int:
self.__monitorTimeout = monitorTimeout
else:
self.__monitorTimeout = 0
def test_files_created(self):
"""Test that we can run experiments and get output files"""
conf_name = 'myconf'
wl_name = 'mywl'
results_dir = os.path.join(self.te.LISA_HOME, 'results', self.res_dir,
'rtapp:{}:{}'.format(conf_name, wl_name))
if os.path.isdir(results_dir):
shutil.rmtree(results_dir)
experiments_conf = {
'confs': [{
'tag': conf_name
}],
"wloads" : {
wl_name : {
"type" : "rt-app",
"conf" : {
"class" : "profile",
"params" : {
"mytask" : {
"kind" : "Periodic",
"params" : {
"duty_cycle_pct": 10,
"duration_s": 0.2,
},
},
},
},
},
},
}
executor = Executor(self.te, experiments_conf)
executor.run()
self.assertTrue(
os.path.isdir(results_dir),
'Expected to find a directory at {}'.format(results_dir))
result_1_dir = os.path.join(results_dir, '1')
self.assertTrue(
os.path.isdir(result_1_dir),
'Expected to find a directory at {}'.format(result_1_dir))
def test_parse(self):
executor = Executor()
parser = executor.parser
parser.functions = executor.get_functions()
test_data = (
"((((((x+1))))+x2))",
"(-1)*x1 + 3*x2 <= 12",
"3+(4+(4-23)+23) <= x^2",
"x+3<= 100",
"x^2 + 3*x + (4 + x*8 + cos(x)) * x <= 100",
# "x^2 + 3*x + 4 + x*8 * x <= 100",
"x^3+x<=99",
"sin(x)==0.6",
"sin(x1)==0.6")
for data in test_data:
print(data + " -> ")
print(parser.parse(data))
def start(self):
pipelines = self.config.get_pipelines()
for (pipeline_name, pipeline) in pipelines.items():
self.get_logger().info("pipelines: %s" % pipeline_name)
executor = Executor(config, pipeline_name, pipeline)
self.executors[pipeline_name] = executor
self.__add_job(pipeline_name, pipeline, executor)
self.__server_run()
class DnsSshCommands(DnsCommands):
def __init__(self, ip_fn):
self.executor = Executor()
self.get_ip = ip_fn
def dns_utility(self, command):
return self.executor.execute_sync_with_output(self._wrap_with_ssh([DNS_UTILITY_COMMAND % command]))
def _wrap_with_ssh(self, command_list):
return wrap_with_ssh(tnt_config.INSTANCE_USER, self.get_ip(), command_list)
def put(self, project_name, task):
""" Put the task to the executor thread associated to the project name.
If the thread does not exist, it will be created.
"""
# Get the executor thread associated to the project name.
executor = self.executors.get(project_name)
if not executor:
# The thread does not exist yet. Create a new one.
executor = Executor()
# Associate this new thread to the project_name.
self.executors[project_name] = executor
# Start the thread.
executor.start()
# Put the task to the good executor thread.
executor.tasks.put(task)
def test_normal_run(self):
def submit_callback(request):
resp = json.loads(request.body.decode("utf-8"))
self.assertEqual(resp['output'], "hello\n")
self.assertEqual(resp['state'], "Executed")
return (200, {}, "Good")
setup_get_writ_resp(get_test_writ())
setup_submit_writ_resp(submit_callback)
Executor(get_test_conf())._run()
def put(self, project_name, task):
""" Put the task to the executor thread associated to the project name.
If the thread does not exist, it will be created.
"""
# Get the executor thread associated to the project name.
executor = self.executors.get(project_name)
if not executor:
# The thread does not exist yet. Create a new one.
executor = Executor()
# Associate this new thread to the project_name.
self.executors[project_name] = executor
# Start the thread.
executor.start()
# Put the task to the good executor thread.
executor.tasks.put(task)
def __init__(self,
cache=False,
cache_ttl=None,
refetch_prob=0,
processor=None):
self.cache = Cache(cache_ttl) if cache else None
self.refetch_prob = refetch_prob
self.processor = processor
self.executor = Executor(num_workers=12)
self._init_session()
def testExpandTemplateExpressions(self):
if IGNORE_TEST:
return
executor = Executor()
var = 'n'
definitions = {var: [1, 2, 3]}
executor.setDefinitions(definitions)
expander = Expander(executor, self.message)
template = "T{%s} + A -> T{%s+1}" % (var, var)
expansion = expander.do(template)
self.assertEqual(len(expansion), len(definitions[var]))
# Check that each substitution is found
for value in definitions[var]:
found = False
for substitution in expansion:
if "T%d" % value in substitution:
found = True
break
self.assertTrue(found)
def main():
"""
Calls the executor to execute a script along with required job
config
"""
if len(sys.argv) < 2:
print 'Missing script name'
sys.exit(1)
elif len(sys.argv) < 3:
print 'Missing job ENVs path'
sys.exit(1)
else:
script_path = sys.argv[1]
job_envs_path = sys.argv[2]
config = Config(script_path, job_envs_path)
ex = Executor(config)
ex.execute()
sys.exit(ex.exit_code)
def do_GET(self):
if not 'controller' in self.headers:
self.server.executor = Executor()
self.send_response(200)
self.end_headers()
return
c = loads(eval(self.headers['controller']))
ans = dumps(self.server.executor.fit(c))
self.send_response(200)
self.end_headers()
self.wfile.write(ans)
def __init__(
self,
host=DEFAULT["host"],
database=DEFAULT["database"],
username=DEFAULT["username"],
password=DEFAULT["password"],
tempDir=DEFAULT["tempDir"],
):
self.liquibaseExecutor = LiquibaseExecutor(host, database, username, password)
self.tempDir = tempDir
self.outputXmlChangeLogFilePath = self.tempDir + "/liquipy_changelog.xml"
def main():
"""
Calls the executor to execute a script along with required job
config
"""
if len(sys.argv) < 2:
print 'Missing script name'
sys.exit(1)
elif len(sys.argv) < 3:
print 'Missing job ENVs path'
sys.exit(1)
else:
script_path = sys.argv[1]
job_envs_path = sys.argv[2]
config = Config(script_path, job_envs_path)
ex = Executor(config)
ex.execute()
sys.exit(ex.exit_code)
def __init__(self,
uri,
exchangeName=None,
exchangeType='direct',
routingKey=None,
otherKeys=[],
applicationId=None,
verbose=False,
subscriberName=None,
recyclebinName=None,
redeliveredLimit=3):
self.__engine = Engine(
**{
'uri': uri,
'exchangeName': exchangeName,
'exchangeType': exchangeType,
'routingKey': routingKey,
'otherKeys': otherKeys,
'applicationId': applicationId,
'verbose': verbose
})
self.__executor = Executor(engine=self.__engine)
self.__listener = None
if subscriberName is not None and type(subscriberName) is str:
self.__subscriberName = subscriberName
self.__executor.assertQueue(self.__subscriberName)
else:
self.__subscriberName = None
if recyclebinName is not None and type(recyclebinName) is str:
self.__recyclebinName = recyclebinName
self.__executor.assertQueue(self.__recyclebinName)
else:
self.__recyclebinName = None
if redeliveredLimit is not None and type(redeliveredLimit) is int:
self.__redeliveredLimit = 0 if (
redeliveredLimit < 0) else redeliveredLimit
else:
self.__recyclebinName = None
def test_run_timelimit(self):
def submit_callback(request):
resp = json.loads(request.body.decode("utf-8"))
self.assertEqual(resp['output'], "Error: Timed out")
self.assertEqual(resp['state'], "TimedOut")
return (200, {}, "Good")
test_writ = get_test_writ()
test_writ['source_code'] = 'import time\ntime.sleep(1000)'
setup_get_writ_resp(test_writ)
setup_submit_writ_resp(submit_callback)
Executor(get_test_conf())._run()
def test_run_outputlimit(self):
def submit_callback(request):
resp = json.loads(request.body.decode("utf-8"))
self.assertEqual(resp['output'], "Error: Output limit exceeded")
self.assertEqual(resp['state'], "OutputLimitExceeded")
return (200, {}, "Good")
test_writ = get_test_writ()
test_writ['source_code'] = 'print("a"*2000)'
setup_get_writ_resp(test_writ)
setup_submit_writ_resp(submit_callback)
Executor(get_test_conf())._run()
def exec_once(self, target):
"""
target: [
{
"ContainerId": "xxx",
"NodeId": "yyy"
},
......
]
"""
e = Executor.new_instance(self, target)
self.executors.append(e.executor_id)
def __init__(self, id, host, port, peerNodeList):
# self.config = config
self.messageBuffer = MessageBuffer()
self.messageLog = MessageLog(
) # TODO: save all messages in Log. currently only the messages sent are saved (the received messages should be saved too!)
self.cryptoHelper = CryptoHelper()
self.executor = Executor()
self.id = id
self.host = host
self.port = port
self.pbftServiceState = PBFTServiceState()
self.peerNodeList = peerNodeList
# N >= 3 f + 1
# N - 1 >= 3 f
# (N - 1)/3 >= f
# and N = len(self.peerNodeList) + 1
self.maxFaultyNodes = len(self.peerNodeList) // 3
if self.maxFaultyNodes < 1:
raise Exception('Not enough peer nodes for BFT provided')
def init_executor(self, cls=None, instance=None):
"""Sets an executor instance/class to be used.
If not called, a default instance of Executor is created.
"""
assert self._executor is None
if instance is None:
if cls is None:
instance = Executor(self)
else:
instance = cls(self)
self._executor = instance
def list_devices(self):
output = Executor.execute(CMD.format('devices'))
os_device_list = json.loads(output)['devices']
device_array = []
for os in os_device_list:
device_array += [Device(device_dict['availability'],
device_dict['name'],
os,
device_dict['state'],
device_dict['udid']
) for device_dict in os_device_list[os]]
return device_array
def main(name,
blocks=[],
modifier_attr_fcts=[],
delta_logging=True,
split_command=None):
# Read the script file and parse the code into an ast
source = open(name, 'r').read()
tree = ast.parse(source)
blocklist = BlockList(*blocks)
# Split the code in different code blocks
# according to the given blocklist
splitter = CodeSplitter(blocklist)
tree_splits, block_flags = splitter.split(tree)
print("Blocks checked:")
print(blocklist.get_blocks())
# Create a Logger wih a given log function and specify the mafs
logger = Logger(log_variable, 'val', 'name', 'lineno', 'db', 'log')
logger.add_modifier_attr_fcts(modifier_attr_fcts)
print("Modifier functions logged:")
print(list(logger.get_modifier_attr_fcts()))
# Visit the ast of the source code and add the needed logging functions
code_splits = []
for i, tree_split in enumerate(tree_splits):
if block_flags[i] == 1:
tree_split = logger.visit(tree_split)
# # Uncomment/Comment this to view/hide
# # source code of created code blocks
# print("-----", i, "-----")
# print(astor.to_source(tree_split))
code_splits.append(compile(tree_split, name, 'exec'))
# Execute the given code blocks
# Specify delta logging and split command as needed
executor = Executor(code_splits, block_flags, delta_logging=delta_logging)
executor.set_split_command(split_command)
executor.run(log_func=log_variable)
log = executor.get_log()
# Print the log after execution
print("Log:")
for log_item in log:
if log_item[1]:
print("{}: Saved {}".format(log_item[0].time.strftime('%H:%M:%S'),
log_item[0]))
else:
print("{}: Not Saved {}".format(
log_item[0].time.strftime('%H:%M:%S'), log_item[0]))
print("Message: {}".format(log_item[2]))
def __init__(self, game):
Group.__init__(self)
self.game = game
self.need_draw = True
self.key_test_period = 0.25
self.__tick = 0
self.rom_executor = RomExecutor()
self.game_list = RomDataItemsConstructor(
game.app.config.get("PATHS", "gamelist"))
self.sdcard_constructor = DirlistItemConstructor(
game.app.config.get("PATHS", "sdcard"), Executor())
self.all_constructors = [
BaseItemConstructor(), # for favorites
self.game_list.getConsole("GEN"), # for gen
self.game_list.getConsole("SMS"), # for sms
self.game_list.getConsole("NES"), # for nes
self.game_list.getConsole("SNES"), # for snes
self.sdcard_constructor,
None
]
self.item_constructor = self.all_constructors[0]
self.bg = Background(game)
self.title_text = TextSprite("", game.assets["TITLE_FONT"])
self.update_title_text(self.game.assets["ICONS"][0]['title'])
self.title_text.centered = True
self.platform = MainMenuPlatformList(game)
self.file_list = FileList(game, self.item_constructor)
self.file_list.deselect_all()
self.selector_state = MainStage.SELECTRO_ICONS
#self.add(*[self.bg, self.title_text, self.platform, self.file_list])
game.app.input.addEvent(input.Input.EVENT_DOWN, self.nextItem)
game.app.input.addEvent(input.Input.EVENT_UP, self.lastItem)
game.app.input.addEvent(input.Input.EVENT_NEXT, self.select)
game.app.input.addEvent(input.Input.EVENT_BACK, self.selectBack)
game.app.input.addEvent(input.Input.EVENT_LEFT, self.last10Item_list)
game.app.input.addEvent(input.Input.EVENT_RIGHT, self.next10Item_list)
self.parts = [self.title_text, self.platform, self.file_list]
def stats_run(n_steps=0):
e = Executor(endo())
#e.debug = True
start = clock()
try:
for i in xrange(2*10**9):
if i > 0 and i%1000 == 0:
print i, int(i/(clock()-start+1e-6)),'steps/s'
e.step()
n_steps -= 1
if n_steps == 0: break
except FinishException:
print 'execution finished'
finally:
print e.iteration, 'iterations'
print len(e.rna), 'rna produced'
print 'it took', clock()-start
print int(e.iteration/(clock()-start+1e-6)), 'iterations/s'
print 'pattern freqs', e.pattern_freqs
print 'template freqs', e.template_freqs
print 'codon len freqs', e.codon_len_freqs
def perform(submission):
result = {"id": submission["id"]}
working_folder = os.path.join(config.PLAYGROUND_PATH, str(os.getpid())) + '/'
if submission.has_key('destination_queue'):
result['start_time'] = time.time()
try:
executor = Executor(submission["lang"].upper(), submission["task"],
submission["source"], working_folder)
(result["result"], result["fail_cause"]) = executor.execute()
except ImportError as error:
result["result"] = "failed"
result["fail_cause"] = "We are sorry, but this language is not available."
if submission.has_key('destination_queue'):
queue = submission['destination_queue']
result['finish_time'] = time.time()
else:
queue = RESQUE_CONFIG['queue_resque']
worker = RESQUE_CONFIG['worker_resque']
ResQ().push(queue, {'class': worker, 'args': [result]})
def load_model(coach_path, executor_path):
if 'onehot' in coach_path:
coach = ConvOneHotCoach.load(coach_path).to(device)
elif 'gen' in coach_path:
coach = RnnGenerator.load(coach_path).to(device)
else:
coach = ConvRnnCoach.load(coach_path).to(device)
coach.max_raw_chars = 200
executor = Executor.load(executor_path).to(device)
executor_wrapper = ExecutorWrapper(coach, executor, coach.num_instructions,
200, 0, 'full')
executor_wrapper.train(False)
return executor_wrapper
def load_model(coach_path, model_path, args):
if 'onehot' in coach_path:
coach = ConvOneHotCoach.load(coach_path).to(device)
elif 'gen' in coach_path:
coach = RnnGenerator.load(coach_path).to(device)
else:
coach = ConvRnnCoach.load(coach_path).to(device)
coach.max_raw_chars = args.max_raw_chars
executor = Executor.load(model_path).to(device)
executor_wrapper = ExecutorWrapper(coach, executor, coach.num_instructions,
args.max_raw_chars, args.cheat)
executor_wrapper.train(False)
return executor_wrapper
def test_run_with_compile_error(self):
def submit_callback(request):
resp = json.loads(request.body.decode("utf-8"))
self.assertIn("SyntaxError: unexpected EOF while parsing",
resp['output'])
self.assertEqual(resp['state'], "Executed")
return (200, {}, "Good")
test_writ = get_test_writ()
test_writ['source_code'] = 'if test:'
setup_get_writ_resp(test_writ)
setup_submit_writ_resp(submit_callback)
Executor(get_test_conf())._run()
def __init__(self):
self.base_path = os.path.dirname(os.path.abspath(__file__))
self.config_path = os.path.abspath(os.path.join(self.base_path, "..", "config.json"))
with open(self.config_path, 'r') as f:
self.configuration = json.load(f)
self.last_message_sent = time()
self.last_ping_sent = time()
self.last_received = None
self.shutdown = False
self.socket = None
self.executor = Executor(self)
self.initialize()
class TrafficDockerCommands(TrafficCommands):
def __init__(self, name):
self.name = name
self.executor = Executor()
def wget(self, command_list):
command = self._wrap_with_docker(command_list)
return self.executor.execute_async(command)
def curl(self, command_list):
command = self._wrap_with_docker(command_list)
return self.executor.execute_async(command)
def cleanup(self, command_list):
command = self._wrap_with_docker(command_list)
return self.executor.execute_sync(command)
def _wrap_with_docker(self, command_list):
command = [
"docker", "exec", "-it", "--workdir", "/node_root/node/", self.name
]
command.extend(command_list)
return command
def main():
parser = argparse.ArgumentParser(description='Produce RNA from DNA')
parser.add_argument('--trace', action='store_true')
parser.add_argument('--pause', action='store_true')
parser.add_argument('prefix_filename')
args = parser.parse_args()
prefix_filename = args.prefix_filename
prefix = open(prefix_filename+'.dna').read().strip()
assert all(c in 'ICFP' for c in prefix)
e = Executor(prefix+endo())
e.debug = args.trace or args.pause
start = clock()
#for r in e.obtain_rna():
# print>>rna, r
try:
while True:
e.step()
if args.pause:
print 'press enter',
raw_input()
except FinishException:
pass
rna_file = open(prefix_filename+'.rna', 'w')
for r in e.rna:
print>>rna_file, r
print 'it took', clock()-start
print int(e.iteration/(clock()-start+1e-6)), 'iterations/s'
rna_file.close()
class FairScheduler:
def __init__(self, cluster_nodes, gpu_per_nodes, daemon=None):
self.E = Executor()
self.daemon = daemon
self.init_job_queue = queue.Queue()
self.resources = build_resource_dict(cluster_nodes, gpu_per_nodes)
self.running_info = queue.Queue()
self.running_jobs = {}
self.growing_jobs = []
self.shrinking_jobs = []
self.msg_handler = threading.Thread(target=self._msg_handle, args=())
self.msg_handler.start()
self.occupy = 0
def _msg_handle(self):
while True:
if self.occupy == 0:
if not self.running_info.empty():
info = self.running_info.get()
info['gpus_loc'] = {'localhost': [0, 1, 2, 3, 4, 5, 6, 7]}
self.occupy = 1
self.E.exec(self.generate_new_job_by_info(info))
@staticmethod
def generate_new_job_by_info(info):
new_job = Job()
new_job.dict_store(info)
return new_job
def queue_empty(self):
if not self.running_info.empty():
return False
else:
return True
def re_enqueue(self, info):
self.running_info.put(info)
def __init__(self,
uri,
exchangeName=None,
routingKey=None,
applicationId=None,
operatorName=None,
responseName=None,
verbose=False):
if logger.isEnabledFor(logging.DEBUG):
logger.debug('Constructor begin ...')
self.__engine = Engine(
**{
'uri': uri,
'exchangeName': exchangeName,
'exchangeType': 'direct',
'routingKey': routingKey,
'applicationId': applicationId,
'verbose': verbose
})
self.__executor = Executor(engine=self.__engine)
if operatorName is not None and type(operatorName) is str:
self.__operatorName = operatorName
self.__executor.assertQueue(self.__operatorName)
else:
raise ConstructorError('operatorName should not be empty')
if responseName is not None and type(responseName) is str:
self.__responseName = responseName
self.__executor.assertQueue(self.__responseName)
else:
self.__responseName = None
self.__middlewares = []
self.__consumerInfo = None
if logger.isEnabledFor(logging.DEBUG): logger.debug('Constructor end!')
class Interpreter:
executor = Executor()
parser = Parser()
def run(self, code: str):
self.parser.load_script(code)
while self.executor.line_n < self.parser.get_line_count():
line = self.parser.get_line(self.executor.line_n)
action, args, p_error = self.parser.parse_line(line, self.executor.actions)
if p_error:
parse_error(self.executor.line_n, p_error)
e_error = self.executor.exec_command(action, args)
if e_error:
execution_error(e_error)
def execute():
"""
Create execution semaphore and threads.
Join threads and returns result files.
"""
resultfiles = []
threads = []
semaphore = threading.BoundedSemaphore(value=Configuration.semaphore)
for (root, dirs, files) in os.walk(Configuration.sourcedir):
for file in files:
# path manipulation
abspath = os.path.join(root, file)
relpath = os.path.relpath(abspath, Configuration.sourcedir)
# Resultdir
resdir = os.path.join(Configuration.resultdir, os.path.dirname(relpath))
# Create resultdir
if not os.path.exists(resdir):
os.makedirs(resdir)
e = Executor(relpath, file, semaphore)
e.start()
threads.append(e)
for t in threads:
t.join()
que = Executor.get_queue()
while que.empty() != True:
resultfiles = resultfiles + que.get()
return resultfiles
def __init__(self):
self.base_path = os.path.dirname(os.path.abspath(__file__))
self.config_path = os.path.abspath(os.path.join(self.base_path, "..", "config.json"))
self.log_path = os.path.abspath(os.path.join(self.base_path, 'logs'))
with open(self.config_path, 'r') as f:
self.configuration = json.load(f)
self.last_message_sent = time()
self.last_ping_sent = time()
self.last_received = None
self.shutdown = False
self.socket = None
self.executor = Executor(self)
self.header = {"User-Agent": "GorillaBot (https://github.com/molly/GorillaBot)"}
self.initialize()
async def test_interface():
ex = Executor("foo")
bot = Bot(ex)
game = Game([bot])
with pytest.raises(NotImplementedError):
await game.play()
with pytest.raises(NotImplementedError):
await game.turn()
with pytest.raises(NotImplementedError):
game.repr_json()
with pytest.raises(NotImplementedError):
game.field_json()
with pytest.raises(NotImplementedError):
game.game_name_json()
with pytest.raises(NotImplementedError):
game.logs_json()
def __init__(
self, rule_file_path, config_file_path, site_config, in_folder_path, out_folder_path, shared_dir, log_file
):
self.rule_file_path = rule_file_path + "/reactor_rules.rrule"
self.config_file_path = config_file_path + "/reactor_config.config"
config_dir_path = site_config[: site_config.rfind("/")]
self.in_folder_path = in_folder_path
self.out_folder_path = out_folder_path
self.shared_dir = shared_dir
self.rule_list = []
self.log_file = log_file
self.buildRules()
self.executor = Executor(config_dir_path)
self.count = 0
self.INVALID_TAGS = ["table", "tbody", "tr", "td"]
self.full_update = os.getenv("REACTOR_UPDATE_ALL") == "YES"
self.update_set = None
def process_conversion(self):
for song in self.files_to_convert:
current_format = ".{}".format(song.rsplit('.', 1)[-1])
old_file = '{}{}{}'.format(self.target_dir, os.sep, song)
new_file = '{}{}{}'.format(self.target_dir, os.sep,
song.replace(current_format, '.mp3'))
message = "Converting {} to 'mp3' format...".format(old_file)
Executor.print_utf(message)
avconv_call = ["avconv", "-i",
old_file.replace(' ', '\ ').replace('|', '\|'),
new_file.replace(' ', '\ ').replace('|', '\|')]
if Executor.execute(avconv_call):
message = 'Deleting {}'.format(old_file)
Executor.print_utf(message)
Executor.execute(["rm", old_file])
def _get_wload(cls, test_env):
cpus = range(test_env.platform['cpus_count'])
# We're pinning stuff in the first phase, so give it ample time to
# clean the pinned logic out of balance_interval
free_time_s = 1.1 * cls._get_max_lb_interval(test_env)
stagger_s = free_time_s / (10 * len(cpus))
params = {}
for cpu in cpus:
params["misfit_{}".format(cpu)] = (
Periodic(
duty_cycle_pct=100,
duration_s=cls.pin_delay_s,
delay_s=cls.idling_delay_s,
period_ms=16,
cpus=[cpu]
) + Periodic(
duty_cycle_pct=100,
# Introduce staggered task completions
duration_s=free_time_s + cpu * stagger_s,
period_ms=16,
cpus=cpus
)
).get()
wload = RTA(test_env.target, 'tmp',
calibration=test_env.calibration())
wload.conf(kind='profile', params=params,
run_dir=Executor.get_run_dir(test_env.target))
return {
'staggered' : {
'type' : 'rt-app',
'conf' : {
'class' : 'custom',
'json' : wload.json
}
}
}
def _read(cls, multi, **kw):
# def _read(cls, multi, sql='', **kw):
kw.setdefault('live', cls.__live__)
kw.setdefault('alias', cls.__alias__)
_sql = kw.pop('sql', '')
cond = kw.pop('condition', '')
if not _sql:
# condition
w1 = (' '.join(cls._basic_fields.get(i[1], i[1]) if i[0] == 1 else i[1] for i in generate_tokens(iter([cond]).next))).replace(': ', ':')
# key=val
w2 = ' AND '.join('%s = :%s' % (cls._basic_fields[i], i) for i in kw if i in cls._basic_fields)
w = w1 + (' AND ' if w1 and w2 else '') + w2
if w: w = 'WHERE ' + w
_sql = 'SELECT %s FROM %s %s' % (','.join(cls._basic_fields.values()), cls.__table__, w)
order = kw.pop('order', cls.__order__)
if multi and order and not cls.__re_ord.search(_sql):
_sql = _sql + ' ORDER BY ' + ' '.join(cls._basic_fields.get(i[1], i[1]) if i[0] == 1 else i[1] for i in generate_tokens(iter([order]).next))
# logging.info(_sql)
return Executor.exec_cls(_sql, cls=cls, multi=multi, **kw)
