def train(model_name): #Get the lables from the .env file labels = env('LABELS') size = int(env('IMG_SIZE')) #create the train data (format the training images) train_data = create_train_data() tf.reset_default_graph() convnet = network(size, labels) model = tflearn.DNN(convnet, tensorboard_dir='log') #if the model already exists, load it so we are not training from scratch if os.path.exists('{}.meta'.format(model_name)): model.load(model_name) print('model loaded!') X = np.array([i[0] for i in train_data]).reshape(-1, size, size, 1) Y = [i[1] for i in train_data] model.fit(X, Y, n_epoch=50) #save the model in the models folder model.save('../models/' + model_name) print("here") print(type(model)) print("here") return model
def test(model_name): labels = env('LABELS') size = int(env('IMG_SIZE')) #format the images that are uploaded test_upload = process_test_data(env('TEST_UPLOAD')) convnet = network(size, labels) model = tflearn.DNN(convnet, tensorboard_dir='log') #load the model for testing model.load('models/' + model_name) for num, data in enumerate(test_upload): img_num = data[1] img_data = data[0] orig = img_data data = img_data.reshape(size, size, 1) model_out = model.predict([data])[0] print(model_out) print(np.argmax(model_out)) str_label = labels[np.argmax(model_out)] #output as a text print(str_label)
def get_base_parser(self): parser = argparse.ArgumentParser( prog='ceilometer', description=__doc__.strip(), epilog='See "ceilometer help COMMAND" ' 'for help on a specific command.', add_help=False, formatter_class=HelpFormatter, ) # Global arguments parser.add_argument('-h', '--help', action='store_true', help=argparse.SUPPRESS, ) parser.add_argument('--version', action='version', version=__init__.__version__) parser.add_argument('-d', '--debug', default=bool(utils.env('MONITORINGCLIENT_DEBUG') ), action='store_true', help='Defaults to env[MONITORINGCLIENT_DEBUG].') parser.add_argument('-v', '--verbose', default=False, action="store_true", help="Print more verbose output.") parser.add_argument('--timeout', default=600, type=_positive_non_zero_int, help='Number of seconds to wait for a response.') parser.add_argument('--ceilometer-url', metavar='<CEILOMETER_URL>', dest='os_endpoint', default=utils.env('CEILOMETER_URL'), help=("DEPRECATED, use --os-endpoint instead. " "Defaults to env[CEILOMETER_URL].")) parser.add_argument('--ceilometer_url', dest='os_endpoint', help=argparse.SUPPRESS) parser.add_argument('--ceilometer-api-version', default=utils.env( 'CEILOMETER_API_VERSION', default='2'), help='Defaults to env[CEILOMETER_API_VERSION] ' 'or 2.') parser.add_argument('--ceilometer_api_version', help=argparse.SUPPRESS) self.auth_plugin.add_opts(parser) self.auth_plugin.add_common_opts(parser) return parser
def stage_upgrade(): sh('rm -rf ${UPGRADE_STAGEDIR}') sh('mkdir -p ${UPGRADE_STAGEDIR}') sh('cp -R ${OBJDIR}/packages/Packages ${UPGRADE_STAGEDIR}/') # If RESTART is given, save that if env('RESTART'): sh('echo ${RESTART} > ${UPGRADE_STAGEDIR}/RESTART') # And if REBOOT is given, put that in FORCEREBOOT if env('REBOOT'): sh('echo ${REBOOT} > ${UPGRADE_STAGEDIR}/FORCEREBOOT') sh('rm -f ${BE_ROOT}/release/LATEST') sh('ln -sf ${UPGRADE_STAGEDIR} ${BE_ROOT}/release/LATEST')
def stage_upgrade(): sh('rm -rf ${UPGRADE_STAGEDIR}') sh('mkdir -p ${UPGRADE_STAGEDIR}') sh('cp -R ${OBJDIR}/packages/Packages ${UPGRADE_STAGEDIR}/') # Move any validation scripts back for v in "ValidateInstall", "ValidateUpdate": if os.path.exists(e('${UPGRADE_STAGEDIR}/Packages/${v}')): sh(e("mv ${UPGRADE_STAGEDIR}/Packages/${v} ${UPGRADE_STAGEDIR}/${v}")) # If RESTART is given, save that if env('RESTART'): sh('echo ${RESTART} > ${UPGRADE_STAGEDIR}/RESTART') # And if REBOOT is given, put that in FORCEREBOOT if env('REBOOT'): sh('echo ${REBOOT} > ${UPGRADE_STAGEDIR}/FORCEREBOOT') sh('rm -f ${BE_ROOT}/release/LATEST') sh('ln -sf ${UPGRADE_STAGEDIR} ${BE_ROOT}/release/LATEST')
def test_debug(model_name): # Only import when called import matplotlib.pyplot as plt labels = env('LABELS') size = int(env('IMG_SIZE')) # if you need to create the data: test_data = process_test_data(env('TEST_DIR')) # if you already have some saved: # test_data = np.load('test_data.npy') convnet = network(size, labels) model = tflearn.DNN(convnet, tensorboard_dir='log') #load the model for testing model.load('../models/' + model_name) fig = plt.figure() #plot the test images along with the label as identified by the model for num, data in enumerate(test_data): img_num = data[1] img_data = data[0] y = fig.add_subplot(2, 2, num + 1) orig = img_data data = img_data.reshape(size, size, 1) model_out = model.predict([data])[0] print(model_out) print(np.argmax(model_out)) str_label = labels[np.argmax(model_out)] y.imshow(orig, cmap='gray') plt.title(str_label) y.axes.get_xaxis().set_visible(False) y.axes.get_yaxis().set_visible(False) plt.show()
def stage_upgrade(): sh('rm -rf ${UPGRADE_STAGEDIR}') sh('mkdir -p ${UPGRADE_STAGEDIR}') sh('cp -R ${OBJDIR}/packages/Packages ${UPGRADE_STAGEDIR}/') # If an update validation script is given, copy that if os.path.exists(e('${PROFILE_ROOT}/ValidateUpdate')): sh('cp ${PROFILE_ROOT}/ValidateUpdate ${UPGRADE_STAGEDIR}/ValidateUpdate') if os.path.exists(e('${PROFILE_ROOT}/ValidateInstall')): sh('cp ${PROFILE_ROOT}/ValidateUpdate ${UPGRADE_STAGEDIR}/ValidateInstall') # Allow the environment to over-ride it -- /dev/null or empty string means # don't have one if env('VALIDATE_UPDATE') is not None: if env('VALIDATE_UPDATE') not in ("/dev/null", ""): sh('cp ${VALIDATE_UPDATE} ${UPGRADE_STAGEDIR}/ValidateUpdate') else: sh('rm -f ${UPGRADE_STAGEDIR}/ValidateUpdate') if env('VALIDATE_INSTALL') is not None: if env('VALIDATE_INSTALL') not in ("/dev/null", ""): sh('cp ${VALIDATE_INSTALL} ${UPGRADE_STAGEDIR}/ValidateInstall') else: sh('rm -f ${UPGRADE_STAGEDIR}/ValidateInstall') # If RESTART is given, save that if env('RESTART'): sh('echo ${RESTART} > ${UPGRADE_STAGEDIR}/RESTART') # And if REBOOT is given, put that in FORCEREBOOT if env('REBOOT'): sh('echo ${REBOOT} > ${UPGRADE_STAGEDIR}/FORCEREBOOT') sh('rm -f ${BE_ROOT}/release/LATEST') sh('ln -sf ${UPGRADE_STAGEDIR} ${BE_ROOT}/release/LATEST')
def stage_upgrade(): sh('rm -rf ${UPGRADE_STAGEDIR}') sh('mkdir -p ${UPGRADE_STAGEDIR}') sh('cp -R ${OBJDIR}/packages/Packages ${UPGRADE_STAGEDIR}/') # If an update validation script is given, copy that if os.path.exists(e('${PROFILE_ROOT}/ValidateUpdate')): sh('cp ${PROFILE_ROOT}/ValidateUpdate ${UPGRADE_STAGEDIR}/ValidateUpdate' ) if os.path.exists(e('${PROFILE_ROOT}/ValidateInstall')): sh('cp ${PROFILE_ROOT}/ValidateUpdate ${UPGRADE_STAGEDIR}/ValidateInstall' ) # Allow the environment to over-ride it -- /dev/null or empty string means # don't have one if env('VALIDATE_UPDATE') is not None: if env('VALIDATE_UPDATE') not in ("/dev/null", ""): sh('cp ${VALIDATE_UPDATE} ${UPGRADE_STAGEDIR}/ValidateUpdate') else: sh('rm -f ${UPGRADE_STAGEDIR}/ValidateUpdate') if env('VALIDATE_INSTALL') is not None: if env('VALIDATE_INSTALL') not in ("/dev/null", ""): sh('cp ${VALIDATE_INSTALL} ${UPGRADE_STAGEDIR}/ValidateInstall') else: sh('rm -f ${UPGRADE_STAGEDIR}/ValidateInstall') # If RESTART is given, save that if env('RESTART'): sh('echo ${RESTART} > ${UPGRADE_STAGEDIR}/RESTART') # And if REBOOT is given, put that in FORCEREBOOT if env('REBOOT'): sh('echo ${REBOOT} > ${UPGRADE_STAGEDIR}/FORCEREBOOT') sh('rm -f ${BE_ROOT}/release/LATEST') sh('ln -sf ${UPGRADE_STAGEDIR} ${BE_ROOT}/release/LATEST')
def main(): """configure and start crawler""" settings = dict( COMPOSE_EMAIL=utils.env('COMPOSE_EMAIL'), COMPOSE_ACCOUNT_NAME=utils.env('COMPOSE_ACCOUNT_NAME'), COMPOSE_PASSWORD=utils.env('COMPOSE_PASSWORD'), COMPOSE_CLUSTER_ID=utils.env('COMPOSE_CLUSTER_ID'), DATADOG_API_KEY=utils.env('DATADOG_API_KEY'), DATADOG_APP_KEY=utils.env('DATADOG_APP_KEY'), DATADOG_TAGS=utils.env('DATADOG_TAGS', ''), USER_AGENT='Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)', ROBOTSTXT_OBEY=True, LOG_LEVEL=utils.env('LOG_LEVEL', 'WARNING'), CHECK_INTERVAL_SECONDS=int(utils.env('CHECK_INTERVAL_SECONDS', 600)), ) configure_logging(settings) loop = task.LoopingCall(crawl, settings=settings) loop.start(settings.get('CHECK_INTERVAL_SECONDS')) reactor.run()
def copy_validators(): # If an update validation script is given, copy that if os.path.exists(e('${PROFILE_ROOT}/ValidateUpdate')): sh('cp ${PROFILE_ROOT}/ValidateUpdate ${pkgdir}/Packages/ValidateUpdate') if os.path.exists(e('${PROFILE_ROOT}/ValidateInstall')): sh('cp ${PROFILE_ROOT}/ValidateUpdate ${pkgdir}/Packages/ValidateInstall') # Allow the environment to over-ride it -- /dev/null or empty string means # don't have one if env('VALIDATE_UPDATE') is not None: if env('VALIDATE_UPDATE') not in ("/dev/null", ""): sh('cp ${VALIDATE_UPDATE} ${pkgdir}/Packages/ValidateUpdate') else: sh('rm -f ${pkgdir}/Packages/ValidateUpdate') if env('VALIDATE_INSTALL') is not None: if env('VALIDATE_INSTALL') not in ("/dev/null", ""): sh('cp ${VALIDATE_INSTALL} ${pkgdir}/Packages/ValidateInstall') else: sh('rm -f ${pkgdir}/Pckages/ValidateInstall') for p in "ValidateUpdate", "ValidateInstall": if os.path.exists(os.path.join(e('${pkgdir}'), "Packages", p)): validators.append(e('-V ${pkgdir}/Packages/' + p))
def create_train_data(): #get the images from the training directory directory = env('TRAIN_DIR') #format the images into numpy array training_data = [] for img in os.listdir(directory): # ignore dot files if (img.startswith('.')): continue label = label_img(img) path = os.path.join(directory, img) img = format_image(path) training_data.append([np.array(img), np.array(label)]) return np.array(training_data)
import utils, requests ISLAND_URL = utils.env("ISLAND_URL", "http://localhost:8000") def generate_file(map_gen_id, file_map_id): resp = requests.post("%s/mapa/generate/%s/file/%s" % (ISLAND_URL, map_gen_id, file_map_id)) if resp.status_code == 200: return resp.json()['id'] raise Exception("Generation failed: [%s] - %s" % (resp.status_code, resp.text))
import utils, json, hvac, logging S_VAULT_HOST = utils.env("S_VAULT_HOST", 'localhost') S_VAULT_SCHEMA = utils.env("S_VAULT_SCHEMA", 'http') S_VAULT_PORT = int(utils.env("S_VAULT_PORT", 8200)) S_VAULT_TOKEN = utils.env("S_VAULT_TOKEN") vault_url = "%s://%s:%i" % (S_VAULT_SCHEMA, S_VAULT_HOST, S_VAULT_PORT) client = hvac.Client(url=vault_url, token=S_VAULT_TOKEN) def put(key, value): logging.info("vault_put: [%s] => [%s]" % (key, value)) secret = json.loads(value) if type(value) == str else value client.secrets.kv.v2.create_or_update_secret(path=key, secret=secret)
import uuid import utils import logging import pystache from contextlib import contextmanager import pyrqlite.dbapi2 as dbapi2 logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) RQLITE_SERVER = utils.env('RQLITE_SERVER', 'localhost') RQLITE_PORT = utils.env('RQLITE_PORT', 4001, int) def get_connection(conn=None): if conn is not None: return conn # Connect to the database _conn = dbapi2.connect( host=RQLITE_SERVER, port=RQLITE_PORT, ) logger.debug("connection opened") return _conn @contextmanager def get_closable_connection():
#! /usr/bin/env python # encoding:utf8 # vim: set ts=4 et sw=4 sts=4 """ File: statistic_words_freqency.py Author: "shanzi" Email: "*****@*****.**" Created At: 2013-05-30 22:31 Description: primary statistic of english words freq take place in tech-concerned website """ from utils import env; env() import re import pymongo from pymongo import MongoClient from Stemmer import Stemmer from searcher.ignorewords import EN as EN_IGNORE from clint.textui import progress,colored EN_WORD_CUT = re.compile(r"[^a-zA-Z]*") client = MongoClient() db = client.fetch_data def run(): stemmer = Stemmer("english") pages = db.en.find() print colored.yellow("statistic words")
import rabbitmq, utils import json, yaml, logging SPRING_PROFILE = utils.env("SPRING_PROFILE", "") # json with prefix/sufix by flavor PREFIX_BAG = utils.env( "PREFIX_BAG", '''{ "spring": { "prefix": "spring/", "sufix": "/data"} }''') _prefix_bag = json.loads(PREFIX_BAG) logging.info("prefix_bag=%s" % str(_prefix_bag)) # json with output format by storage (you can have a .yaml that seed consul as json) STORAGE_OUTPUT = utils.env("STORAGE_OUTPUT", '''{ "consul": "yaml", "vault": "json" }''') _storage_output = json.loads(STORAGE_OUTPUT) logging.info("storage_outputs=%s", str(_storage_output)) exchange = utils.env('EXCHANGE', 'seeder') queue_name = utils.env('QUEUE', 'seeder-transform') input_routing_key = utils.env('INPUT_RK', 'transform') output_routing_key = utils.env('OUTPUT_RK', 'seed') conn = rabbitmq.get_connection() rabbitmq.create_exchange(exchange, conn) rabbitmq.create_queue(queue_name, exchange, input_routing_key, conn) def load_custom_transformer(): tb, ta = None, None try:
@app.route("/log", methods=['POST']) async def log(request): logger.info("received => [%s]" % request.body) return text('OK') @app.route("/rabbitmq/<exchange>/<routing_key>", methods=['POST']) async def log(request, exchange, routing_key): rabbitmq.publish(exchange, routing_key, request.body) return text('OK') @app.route("/health") async def health(request): return text('OK') if __name__ == "__main__": import utils APP_PORT = int(utils.env("APP_PORT", 8000)) APP_WORKERS = int(utils.env("APP_WORKERS", 1)) exchange = utils.env('EXCHANGE') if exchange: rabbitmq.create_exchange(exchange) app.run(host="0.0.0.0", port=APP_PORT, workers=APP_WORKERS)
if poudriere_proc and poudriere_proc.pid: try: os.kill(poudriere_proc.pid, signal.SIGTERM) except OSError: pass def cleanup_gui(): # FIXME: This triggers a kernel panic possible because of long path names # with spaces and double dashes in it. info('Cleaning up gui modules') sh('rm -rf ${BE_ROOT}/gui/node_modules') if __name__ == '__main__': if env('SKIP_PORTS'): info('Skipping ports build as instructed by setting SKIP_PORTS') sys.exit(0) cleanup_gui() create_overlay() on_abort(cleanup_env) obtain_jail_name() calculate_make_jobs() create_poudriere_config() create_make_conf() create_ports_list() prepare_jail() merge_port_trees() prepare_env() signal.signal(signal.SIGTERM, terminate)
@app.route("/mapa/<id>/templars", methods=['GET']) async def get_mapa(request, id): return json(mapa.get_templars(id)) @app.route("/mapa/<id>", methods=['DELETE']) async def delete(request, id): mapa.delete(id) return text('OK') @app.route("/mapa/<id>/generate", methods=['POST']) async def delete(request, id): mapa.start_map_generation(id, request.json) return text('OK') @app.route("/mapa/generate/<map_gen_id>/file/<file_map_id>", methods=['POST']) async def delete(request, map_gen_id, file_map_id): return json({'id': mapa.generate_file(map_gen_id, file_map_id)}) if __name__ == "__main__": APP_PORT = utils.env("APP_PORT", 8000, int) APP_WORKERS = utils.env("APP_WORKERS", 2, int) app.run(host="0.0.0.0", port=APP_PORT, workers=APP_WORKERS)
import json import utils import requests TEMPLAR_URL = utils.env("TEMPLAR_URL", "http://localhost:8001") def compile(id, config): resp = requests.post("%s/%s/compile" % (TEMPLAR_URL, id), data=json.dumps(config)) if resp.status_code == 200: return resp.text raise Exception("Failed to compile template '%s. %s - %s" % (id, resp.status_code, resp.text))
import utils import consul, logging S_CONSUL_HOST = utils.env("S_CONSUL_HOST", "localhost") S_CONSUL_PORT = int(utils.env("S_CONSUL_PORT", 8500)) c = consul.Consul(host=S_CONSUL_HOST, port=S_CONSUL_PORT, scheme='http') def put(id, value): logging.info("consul_put: [%s] => [%s]" % (id, value)) c.kv.put(id, value)
import json, logging import rabbitmq, utils import consul_client, vault_client exchange = utils.env('EXCHANGE', 'seeder') queue_name = utils.env('QUEUE', 'seed') input_routing_key = utils.env('INPUT_RK', 'seed') output_routing_key = utils.env('OUTPUT_RK', 'seeded') conn = rabbitmq.get_connection() rabbitmq.create_exchange(exchange, conn) rabbitmq.create_queue(queue_name, exchange, input_routing_key, conn) def notify_seed_app(app): rabbitmq.publish(exchange, output_routing_key, {'app': app}, conn) def seed(message): key = message['key'] value = message['value'] storage = message['storage'] app = message['app'] if storage == 'consul': consul_client.put(key, value) notify_seed_app(app) elif storage == 'vault': vault_client.put(key, value)
except Exception: raise ValueError('YAML value is expected, but got: %s' % value) return value def __repr__(self): return "YAML data" def generate_output_name(): file_name = "shaker_%s.json" % utils.strict(str(datetime.datetime.now())) return os.path.join(tempfile.gettempdir(), file_name) COMMON_OPTS = [ cfg.Opt('server-endpoint', default=utils.env('SHAKER_SERVER_ENDPOINT'), required=True, type=Endpoint(), help='Address for server connections (host:port), ' 'defaults to env[SHAKER_SERVER_ENDPOINT].'), cfg.IntOpt('polling-interval', default=(utils.env('SHAKER_POLLING_INTERVAL') or DEFAULT_POLLING_INTERVAL), help='How frequently the agent polls server, in seconds') ] OPENSTACK_OPTS = [ cfg.StrOpt('os-auth-url', metavar='<auth-url>', default=utils.env('OS_AUTH_URL'), sample_default='',
import rabbitmq, utils import json, logging, requests # load configs exchange = utils.env('EXCHANGE', 'rabbitmqp') output_exchange = utils.env('OUTPUT_EXCHANGE') queue = utils.env('QUEUE', 'my-queue') input_rk = utils.env('INPUT_RK', 'input') output_rk = utils.env('OUTPUT_RK') post_url = utils.env('POST_URL') # get rabbit connection conn = rabbitmq.get_connection() # create exchanges rabbitmq.create_exchange(exchange, conn) if output_exchange and output_exchange != exchange: rabbitmq.create_exchange(output_exchange, conn) # create queue's and bind's rabbitmq.create_queue(queue, exchange, input_rk, conn) # message consumer callback def callback(ch, method, properties, body): # call backend if post_url is not None: logging.info("Posting message to url %s" % post_url) resp = requests.post(post_url, data=body)
import os import utils import requests API_URL=utils.env("API_URL", "http://localhost:8000") def get_file(id): return requests.get("%s/storage/file/%s" % (API_URL, id)) def delete_file(id): return requests.delete("%s/storage/file/%s" % (API_URL, id)) def k8s_bounce_app(app): return requests.get("%s/k8s/bounce/%s" % (API_URL, app))
from sanic import Sanic from sanic.log import logger from sanic.response import json from sanic.response import text import os import fstore import rabbitmq, utils exchange = utils.env('EXCHANGE', 'seeder') output_routing_key = utils.env('OUTPUT_RK', 'split') KUBE_BOUNCE_COMMAND = utils.env( 'KUBE_BOUNCE_COMMAND', '''kubectl set env deployment/%s --env="LAST_MANUAL_RESTART=$(date +%%s)"''' ) rabbitmq.create_exchange(exchange) app = Sanic() @app.route("/seed/<store>/<flavor>", methods=['POST']) async def seed(request, store, flavor): id = fstore.save(request.files["file"][0].body) extension = request.files["file"][0].name.split('.')[-1] name = request.files["file"][0].name.split('.')[0] message = { 'id': id, 'storage': store, 'name': name, 'type': extension if extension not in ['yaml', 'yml'] else 'yaml', 'flavor': flavor
node_modules = e('${GUI_STAGEDIR}/node_modules') os.chdir(e('${GUI_STAGEDIR}')) sh('npm install') sh('${node_modules}/.bin/gulp deploy --output=${GUI_DESTDIR}', log=logfile, mode='a') def create_plist(): with open(e('${GUI_DESTDIR}/gui-plist'), 'w') as f: for i in walk('${GUI_DESTDIR}'): if not os.path.isdir(e('${GUI_DESTDIR}/${i}')): f.write(e('/usr/local/www/gui/${i}\n')) with open(e('${GUI_STAGEDIR}/custom-plist')) as c: f.write(c.read()) if __name__ == '__main__': if env('SKIP_GUI'): info('Skipping GUI build as instructed by setting SKIP_GUI') sys.exit(0) info('Building GUI') info('Log file: {0}', logfile) cleandirs() copy() install() create_plist()
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # ##################################################################### import os import sys from dsl import load_profile_config from utils import sh, sh_str, env, e, objdir, info, debug, pathjoin config = load_profile_config() arch = env('TARGET_ARCH', 'amd64') makeconfbuild = objdir('make-build.conf') kernconf = objdir(e('${KERNCONF}')) kernconf_debug = objdir(e('${KERNCONF}-DEBUG')) kernlog = objdir('logs/buildkernel') kerndebuglog = objdir('logs/buildkernel-debug') worldlog = objdir('logs/buildworld') makejobs = None def calculate_make_jobs(): global makejobs jobs = sh_str('sysctl -n kern.smp.cpus') if not jobs: makejobs = 2
import logging, json import utils, rabbitmq, island logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) # rabbit config exchange = utils.env('EXCHANGE', 'octopus') queue_name = utils.env('QUEUE', 'island_file_gen') routing_key = utils.env('INPUT_RK', 'file_generate') rabbit_conn = rabbitmq.get_connection() rabbitmq.create_exchange(exchange, rabbit_conn); rabbitmq.create_queue(queue_name, exchange, routing_key, rabbit_conn) def file_generate_callback(ch, method, properties, body): logger.info("Generating file [%s]" % body) p_body = json.loads(body) map_gen_id = p_body['map_gen_id'] file_map_id = p_body['file_map_id'] try: gen_id = island.generate_file(map_gen_id, file_map_id) logger.info("map_gen_file_id: %s", gen_id) except Exception as e: logger.error(e) rabbitmq.send_to_retry("Failed", properties) rabbitmq.consumer(queue_name, file_generate_callback, rabbit_conn)
import os, pika, json import utils, logging MQ_HOST = utils.env("MQ_HOST", "localhost") MQ_PORT = int(utils.env("MQ_PORT", 5672)) MQ_USER = utils.env("MQ_USER", "guest") MQ_PASS = utils.env("MQ_PASS", "guest") MQ_VHOST = utils.env("MQ_VHOST", "/") MQ_TTL_RETRY = int(utils.env("MQ_TTL_RETRY", 30000)) # 30s MQ_TTL_LONG_RETRY = int(utils.env("MQ_TTL_LONG_RETRY", 10 * MQ_TTL_RETRY)) # 10x30s = 300s = 5min MQ_MAX_RETRIES = int(utils.env("MQ_MAX_RETRIES", 10)) # 5min retries MQ_MAX_LONG_RETRIES = int(utils.env("MQ_MAX_LONG_RETRIES", 10)) # + 50min long retries MQ_PRE_FETCH = int(utils.env("MQ_PRE_FETCH", 1)) credentials = pika.PlainCredentials(MQ_USER, MQ_PASS) connection_params = pika.ConnectionParameters(MQ_HOST, MQ_PORT, MQ_VHOST, credentials) class RetryException(Exception): pass class LongRetryException(Exception): pass def get_connection():
import logging, json import utils, api_client, rabbitmq exchange = utils.env('EXCHANGE', 'seeder') queue_name = utils.env('QUEUE', 'seeder-split') input_routing_key = utils.env('INPUT_RK', 'split') output_routing_key = utils.env('OUTPUT_RK', 'transform') conn = rabbitmq.get_connection() rabbitmq.create_exchange(exchange, conn) rabbitmq.create_queue(queue_name, exchange, input_routing_key, conn) def callback(ch, method, properties, body): p_body = json.loads(body) id = p_body['id'] type = p_body['type'] message = dict(p_body) # get file by id resp = api_client.get_file(id) if resp.status_code != 200: error_msg = "Failed to get file %s. [HTTP_STATUS=%i] - %s" % ( id, resp.status_code, resp.text) rabbitmq.send_to_retry(error_msg, properties) raw_data = resp.text logging.info("raw_file=[%s]" % raw_data) if type == 'yaml':
0: ("models/ob_rms/osco.pt", craft.OneStoppedCarOEnv), 1: ("models/ob_rms/osc.pt", craft.OneStoppedCarEnv), 2: ("models/ob_rms/2sc.pt", craft.TwoStoppedCarsEnv), 3: ("models/ob_rms/3sc.pt", craft.ThreeStoppedCarsSSO), } num_episodes = 10 render = False discrete = False device = torch.device("cuda" if torch.cuda.is_available() else "cpu") intervals = 10 all_buckets = {} bmax = 0 plt.figure(figsize=(10, 6)) for env_id, (model_name, env) in all_envs.items(): eval_env = env(discrete=discrete) eval_env.debug['action_buckets'] = True if not os.path.isfile(model_name): print("Not trained, run with -train") exit(0) policy = utils.torch.load_empty_policy(learn.PolicyPPO, "models/gym_spaces.pt", hidden=64) ob_rms = policy.load_model(model_name) utils.env.evaluate_ppo(policy, ob_rms, eval_env, device, num_episodes=num_episodes, render=render, discrete=discrete)
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # ##################################################################### import os import sys from dsl import load_profile_config from utils import sh, sh_str, env, e, setup_env, objdir, info, debug, error, pathjoin config = load_profile_config() arch = env('TARGET_ARCH', 'amd64') makeconfbuild = objdir('make-build.conf') kernconf = objdir(e('${KERNCONF}')) kernconf_debug = objdir(e('${KERNCONF}-DEBUG')) kernlog = objdir('logs/buildkernel') kerndebuglog = objdir('logs/buildkernel-debug') worldlog = objdir('logs/buildworld') makejobs = None def calculate_make_jobs(): global makejobs jobs = sh_str('sysctl -n kern.smp.cpus') if not jobs: makejobs = 2