def main(gcm="", rcm="", out_folder=""):
pc = None
pf = None
kwargs = {
"start_year": 1970,
"end_year": 1999,
"rcm": rcm, "gcm": gcm,
"out_folder": out_folder
}
in_folder = "data/narccap/{0}-{1}/current".format(gcm, rcm)
if os.path.isdir(in_folder):
pc = Process(target=interpolate_to_amno, args=(in_folder, ), kwargs=kwargs)
pc.start()
else:
print "{0} does not exist, ignoring the period ...".format(in_folder)
kwargs = {
"start_year": 2041,
"end_year": 2070,
"rcm": rcm, "gcm": gcm,
"out_folder": out_folder
}
in_folder = "data/narccap/{0}-{1}/future".format(gcm, rcm)
if os.path.isdir(in_folder):
pf = Process(target=interpolate_to_amno, args=(in_folder, ), kwargs=kwargs)
pf.start()
else:
print "{0} does not exist, ignoring the period ...".format(in_folder)
#do current and future climates in parallel
if pc is not None: pc.join()
if pf is not None: pf.join()
def lookup(self, buildingID):
from building.dummy import dummy
'''
@buildingID: int
'''
building = Building.objects.get(bid=buildingID)
print building
if building is None:
return None
if self.binstances.has_key(buildingID) is False:
ins = None
if building.bri == BUILDING_REPRESENTATION_DEBUG:
ins = dummy.DummyBif(buildingID)
elif building.bri == BUILDING_REPRESENTATION_IFC:
raise NotImplementedError, 'Missing implementation of IFC plugin'
elif building.bri == BUILDING_REPRESENTATION_REVIT:
raise NotImplementedError, 'Missing implementation of Revit plugin'
# Fire up the simulator for the building if necessary.
self.comm_queues[buildingID] = Queue()
p = Process(target=_Manager__init_simulator, args=(ins, self.comm_queues[buildingID]))
p.start()
self.binstances[buildingID] = ins
return self.binstances[buildingID]
def process_input_dir(args, input_path, output_path):
patt = input_path + os.sep + "*" + args.extension
files = glob.glob(patt)
docs_num = len(files)
if docs_num > args.threads:
slice_size = docs_num / args.threads
else:
slice_size = 1
print "Threads:", args.threads
print "Documents number:", docs_num
print "Documents per thread:", slice_size
start = 0
jobs = []
for job_num in range(args.threads):
print "Initializing process", job_num
end = start + slice_size
p = Process(target=lemmatize_files, args=(files[start:end], output_path, args))
print files[start:end]
jobs.append(p)
p.start()
start += slice_size
for p in jobs:
p.join()
if (docs_num % 2) == 1:
lemmatize_files(files, output_path, args)
def process(host,port):
from multiprocessing.process import Process
p = Process(target=run, args=(host,port))
p.daemon = True
p.start()
return p
def service_background_jobs(self):
logger.info('service_background_jobs')
# NOTE: paths must begin with a "/", indicating that the first part of
# the URI is a script name (which each app, i.e "reports" serves as).
# see django.core.handlers.wsgi.__init__
uri = '/' + '/'.join([SCHEMA.REPORTS_API_URI, JOB.resource_name ])
data = {
JOB.STATE: SCHEMA.VOCAB.job.state.PENDING }
kwargs = {}
logger.info('get jobs: %r', uri)
resp = self.api_client.get(
uri, data=data, **kwargs)
job_listing = self.api_client.deserialize(resp)
if API_RESULT_DATA in job_listing:
job_listing = job_listing[API_RESULT_DATA]
for job in job_listing:
logger.info('found job: %r', job)
job_id =job[JOB.ID]
logger.info('Process the job: %r', job_id)
p = Process(target=self.client_processor.service,args=(job_id,) )
# make the parent process wait:
# p.daemon = True
# if set to true, then the parent process won't wait.
logger.info('start')
p.start();
logger.info('started...')
logger.debug('servicing completed')
def __init__(self, fd, queue):
assert hasattr(queue, 'put')
assert hasattr(queue, 'empty')
assert callable(fd.readline)
Process.__init__(self)
self._fd = fd
self._queue = queue
def async_file_reading(fd, callback):
"""Helper which instantiate and run an AsynchronousFileReader."""
queue = SimpleQueue()
reader = AsynchronousFileReader(fd, queue)
reader.start()
consummer = Process(target=consume_queue, args=(queue, callback))
consummer.start()
return (reader, consummer)
def __init__(self, accessLevel, eightApi):
self.accessLevel = accessLevel
self._eightApi = eightApi
pygame.init()
self.check()
thread = Process(target=self.inputHandler)
thread.start()
def __init__(self, pipe, object_class, *args, **kwargs):
Process.__init__(self)
self.object_class = object_class
self.init_args = args
self.init_kwargs = kwargs
self.pipe = pipe
self.command_queue = []
self.message_priorities = {}
def add_server(self, app_name, service_name, host, port, processor, use_simple_server=True, wait=1,
use_ssl=False, ca_certs=None, cert=None, key=None):
self.sd_client.register_endpoint(app_name, service_name, host, port)
server_process = Process(target=self.__thrift_server,
args=(processor, host, port, use_simple_server, use_ssl, ca_certs, cert, key))
server_process.start()
time.sleep(wait)
self.server_processes.append(server_process)
def __init__(self, sem, port=2022, protocol=ProtocolEnumerator.ECHO, users={}):
Process.__init__(self, name = "SSHServer")
self.port = port
self.users = users
self.privateKeys = {}
self.publicKeys = {}
self.protocol_ = protocol
self.exitCmd = "exit"
self.messages = []
self.sem = sem
def __init__(self, event_loop = ThreadedEventLoop()):
Process.__init__(self)
self.server_pipe, self.client_pipe = Pipe()
self.server_router = Router(self.server_pipe)
self.client_router = Router(self.client_pipe)
self.srv_command_pipe = self.server_router.create()
self.cli_command_pipe = self.client_router.create(self.srv_command_pipe.id)
self.children = {}
self.client_event_loop = event_loop
self.client_event_loop.register_hook(self.client_router.eventloop_hook)
def __init__(self, **kwargs):
"""
@param init_target: unbound instance method to call at process start
@param wrap_target: unbound instance method to wrap worker function
@param term_target: unbound instance method to call at process finish
"""
self._init_target = kwargs.pop("init_target", lambda self: None).__get__(self, Process)
self._wrap_target = kwargs.pop("wrap_target", lambda self: None).__get__(self, Process)
self._term_target = kwargs.pop("term_target", lambda self: None).__get__(self, Process)
Process.__init__(self, **kwargs)
def test_mcdpweb_server(dirname):
port = random.randint(11000, 15000)
base = 'http://127.0.0.1:%s' % port
p = Process(target=start_server, args=(dirname, port,))
p.start()
print('sleeping')
time.sleep(5)
try:
url_wrong = base + '/not-existing'
urllib2.urlopen(url_wrong).read()
except HTTPError:
pass
else:
raise Exception('Expected 404')
# now run the spider
tmpdir = tempfile.mkdtemp(prefix='wget-output')
cwd = '.'
cmd = ['wget', '-nv', '-P', tmpdir, '-m', base]
# res = system_cmd_result(
# cwd, cmd,
# display_stdout=True,
# display_stderr=True,
# raise_on_error=True)
sub = subprocess.Popen(
cmd,
bufsize=0,
cwd=cwd)
sub.wait()
exc = get_exceptions(port)
if len(exc) == 0:
msg = 'Expected at least a not-found error'
raise Exception(msg)
if not 'not-existing' in exc[0]:
raise Exception('Could not find 404 error')
exc = exc[1:]
if exc:
msg = 'Execution raised errors:\n\n'
msg += str("\n---\n".join(exc))
raise_desc(Exception, msg)
url_exit = base + '/exit'
urllib2.urlopen(url_exit).read()
print('waiting for start_server() process to exit...')
p.join()
print('...clean exit')
def __init__(self):
self.__actualSpeed = Value('f', 0.0)
self._targetSpeed = Value('f', 0.0)
self.__actualDirection = Value('f', 0.0)
self._targetDirection = Value('f', 0.0)
self.__setupMotors()
steering = Process(target=self.steer)
#steering.daemon = True
steering.start()
def __new__(self, type_, tId, qDistributor, q, model):
if type_==SolverImplType.THREADED:
thread = threading.Thread(target=SolverWorker, args=[tId, q, qDistributor, model])
thread.setName("LoopingSolver_thread_%(I)s"%{"I":tId})
thread.setDaemon(True)
elif type_==SolverImplType.MULTIPROCESSOR:
thread = Process(target=SolverWorker, args=[tId, q, qDistributor, model])
thread.setName("LoopingSolver_process_%(I)s"%{"I":tId})
thread.setDaemon(True)
else:
raise TypeError("Unknown SolverImplType: '%(T)s'."%{"T":type_})
return thread
def import_library_in_another_process(path, args):
q = Queue(maxsize=1)
p = Process(target=library_initializer, args=(q, path, args))
p.start()
while True:
try:
result = q.get(timeout=0.1)
if isinstance(result, Exception):
raise ImportError(result)
return result
except Empty:
if not p.is_alive():
raise ImportError()
def setUp(self):
# Use Thread for debug:
if USE_THREADS:
self.server = Thread(target=ServerLoop, args=(self.container_name,))
else:
self.server = Process(target=ServerLoop, args=(self.container_name,))
self.server.start()
self.count = 0
self.data_arrays_sent = 0
time.sleep(0.1) # give it some time to start
self.rdaemon = Pyro4.Proxy("PYRO:Pyro.Daemon@./u:"+self.container_name)
self.comp = self.rdaemon.getObject("mycomp")
def __init__(self, config, logger=None, logname='imagemngr'):
"""
Create an instance of the image manager.
"""
if logger is None and logname is None:
self.logger = logging.getLogger(logname)
log_handler = logging.StreamHandler()
logfmt = '%(asctime)s [%(name)s] %(levelname)s : %(message)s'
log_handler.setFormatter(logging.Formatter(logfmt))
log_handler.setLevel(logging.INFO)
self.logger.addHandler(log_handler)
elif logname is not None:
self.logger = logging.getLogger(logname)
self.logger.info('ImageMngr using logname %s' % (logname))
else:
print "Using upstream logger"
self.logger = logger
print logger
self.logger.info('ImageMngr using upstream logger')
self.logger.debug('Initializing image manager')
self.config = config
if 'Platforms' not in self.config:
raise NameError('Platforms not defined')
self.systems = []
# Time before another pull can be attempted
self.pullupdatetimeout = 300
if 'PullUpdateTime' in self.config:
self.pullupdatetimeout = self.config['PullUpdateTimeout']
# Max amount of time to allow for a pull
self.pulltimeout = self.pullupdatetimeout
# This is not intended to provide security, but just
# provide a basic check that a session object is correct
self.magic = 'imagemngrmagic'
if 'Authentication' not in self.config:
self.config['Authentication'] = "munge"
self.auth = Authentication(self.config)
self.platforms = self.config['Platforms']
for system in self.config['Platforms']:
self.systems.append(system)
# Connect to database
if 'MongoDBURI' not in self.config:
raise NameError('MongoDBURI not defined')
self.workers = WorkerThreads()
self.status_queue = self.workers.get_updater_queue()
self.status_proc = Process(target=self.status_thread,
name='StatusThread')
self.status_proc.start()
atexit.register(self.shutdown)
self.mongo_init()
def main():
''' parse the command line - new up the appl and listen on port '''
if os.path.isfile("../kew_pe.conf"):
print ("Loading config file ../kew_pe.conf")
options.parse_config_file("../kew_pe.conf")
options.parse_command_line()
logging.basicConfig(level=logging.DEBUG)
#report_control.process('localhost',8081)
process = Process(target=report_control.run, name="report_control", kwargs={'host':'localhost', 'port':8081})
process.daemon = True
process.start()
def test_litmus_with_authentication(self):
"""Run litmus test suite on HTTP with authentification.
This test passes
"""
try:
proc = Process(target=run_wsgidav_server, args=(True, False))
proc.daemon = True
proc.start()
time.sleep(1)
try:
self.assertEqual(subprocess.call(["litmus", "http://127.0.0.1:8080/", "tester", "secret"]),
0,
"litmus suite failed: check the log")
except OSError:
print "*" * 70
print "This test requires the litmus test suite."
print "See http://www.webdav.org/neon/litmus/"
print "*" * 70
raise
finally:
proc.terminate()
proc.join()
def start_sched():
j_logger.info(u'starting job scheduler ...')
jobs = get_all_jobs()
for job in jobs:
j_logger.info(u'starting job %s ' % job.name)
job.run_status = Value('i', 0) #job的状态值
try:
p = Process(target=_inner_job, name=job.name,args=(job,))
p.start()
job.process_id = p.pid
j_logger.info(u'job %s started !' % job.name)
except Exception as e:
j_logger.error(u'job %s fail to start,due to [%s]!' % (job.name,e))
register_signal_notify()
j_logger.info(u'job scheduler started !')
def start_sched():
j_logger.info(u'starting job scheduler ...')
jobs = get_all_jobs()
for job in jobs:
j_logger.info(u'starting job %s ' % job.name)
job.run_status = Value('i', 0) #job的状态值
try:
p = Process(target=_inner_job, name=job.name, args=(job, ))
p.start()
job.process_id = p.pid
j_logger.info(u'job %s started !' % job.name)
except Exception as e:
j_logger.error(u'job %s fail to start,due to [%s]!' %
(job.name, e))
register_signal_notify()
j_logger.info(u'job scheduler started !')
def setUp(self):
# Use Thread for debug:
if USE_THREADS:
self.server = threading.Thread(target=ServerLoop,
args=(self.container_name, ))
else:
self.server = Process(target=ServerLoop,
args=(self.container_name, ))
self.server.start()
self.count = 0
self.data_arrays_sent = 0
time.sleep(0.1) # give it some time to start
self.rdaemon = Pyro4.Proxy("PYRO:Pyro.Daemon@./u:" +
self.container_name)
self.comp = self.rdaemon.getObject("mycomp")
def test_wsgi_404(self):
def run_provider(queue):
try:
provider = create_provider()
app = Application(provider=provider)
httpd = make_server('', 15486, app,
handler_class=NoLoggingHandler)
queue.put({"result": 0})
httpd.serve_forever()
except Exception as e:
queue.put({"result": 1, "error_message": str(e)})
ready_queue = Queue()
self.server = Process(target=run_provider, args=(ready_queue,))
self.server.start()
provider_started = ready_queue.get()
if provider_started["result"] != 0:
raise Exception("Error starting Provider process with message"
"'{0}'".format(provider_started["error_message"]))
try:
urlopen("http://127.0.0.1:15486/invalid-path").read()
except HTTPError as e:
self.assertEqual(404, e.code)
def main():
auth_server = Process(target=run_auth_server)
auth_server.start()
print("To test getting an auth token, execute the following curl command:")
print(
"curl --ipv4 -v -X POST"
" -d 'grant_type=client_credentials&client_id=abc&client_secret=xyz' "
"http://localhost:8080/token"
)
def sigint_handler(signal, frame):
print("Terminating server...")
auth_server.terminate()
auth_server.join()
signal.signal(signal.SIGINT, sigint_handler)
def run(self): self._init_target() if self._wrap_target: self._target = self._wrap_target(self._target) try: return Process.run(self) finally: self._term_target()
def _start(self,name,cpu, module_name, class_name, params):
fn = None
self._processes = []
self._in_queue = Queue()
self._out_queue = Queue()
self._log_queue = Queue()
if name == "mapper":
fn = q_run_mapper
elif name == "reducer":
fn = q_run_reducer
for i in range(cpu):
process = Process(target=fn,args=(module_name, class_name ,params, self._in_queue, self._out_queue, self._log_queue))
self._processes.append(process)
process.start()
def set_from_file(self, varfile_path, args):
q = Queue()
p = Process(target=set_from_file, args=(q, varfile_path, args))
p.start()
p.join()
there_are_results = False
while True:
try:
results = q.get_nowait()
there_are_results = True
if len(results) == 1:
raise DataError(results[0])
self.set(*results)
except Empty:
if not there_are_results:
raise DataError('No variables')
return
def __enter__(self):
self.smtp_process_queue = Queue()
self.smtp_process = Process(
target=get_otp_mail, args=(self.smtp_process_queue, self.timeout))
self.smtp_process.start()
self.port = self.smtp_process_queue.get(True, 5)
self._do_lintop_config()
return self
def multiprocess_map(target,params, *args):
mgr = multiprocessing.Manager()
dict_threadsafe = mgr.dict()
jobs = [Process(target=target_wrapper, args=(target,param,index,dict_threadsafe,args)) for index, param in enumerate(params)]
for job in jobs: job.start()
for job in jobs: job.join()
return dict_threadsafe.values()
class FileResultsLogger(ResultsLogger):
process = None
queue = None
logFileName = None
def __init__(self, args):
self.queue = Queue()
if args.no_p2p:
fileName = '_'.join([args.mem_algo, str(args.mem_size), 'no_p2p'])
elif args.no_mem:
fileName = '_'.join(
[args.p2p_net, 'no_mem', args.p2p_algo,
str(args.p2p_size)])
else:
fileName = '_'.join([
args.p2p_net, args.mem_algo,
str(args.mem_size), args.p2p_algo,
str(args.p2p_size)
])
self.logFileName = 'logs/' + fileName + '.logs'
def logRequest(self, nodeId, address, latency, cacheLevel=-1):
ResultsLogger.logRequest(self,
nodeId,
address,
latency,
cacheLevel=cacheLevel)
self.queue.put("{}\t{}\t{}\t{}\n".format(nodeId, address, latency,
cacheLevel))
def finish(self):
self.queue.put("STOP")
self.process.join()
def writerProcess(self):
fileObject = open(self.logFileName, 'w+')
for logEntry in iter(self.queue.get, "STOP"):
fileObject.write(logEntry)
fileObject.close()
def start(self):
self.process = Process(target=self.writerProcess)
self.process.start()
def load_models(self, pmodels):
"""assumes the pmodels length is equivalent to the number of desired processes."""
if self.use_lock:
lock = Lock()
else:
lock = None
processes = [
Process(target=self.loader, args=(pm, lock)) for pm in pmodels
]
self.run(processes)
def __init__(self, username, password, host, cmdsToSend, port = 22, exitCmd = "exit", timeout = None):
Process.__init__(self, name = "SSHClient")
self.logger = LogManager().getLogger('SSHClient-%s' % host)
self.username = username
self.password = password
self.host = host
self.port = int(port)
self.cmdsToSend = cmdsToSend if isinstance(cmdsToSend, list) else [cmdsToSend]
self.exitCmd = exitCmd
self.queue = Queue()
self.msg = ""
self.status = Status.FAILURE
self.startTime = Value('d', 0.0)
self.endTime = Value('d', 0.0)
self.timeout = timeout or SSHClient.TIMEOUT
self.cmdsSend = False
self.start()
def forecast_demands(self):
""" See :meth:`StatisticalForecast.forecast_demands`.
This method uses processes, to speed up the calculation.
This drives the cpu to full load for a short time."""
cached = StatisticalForecast.read_from_cache(self)
if cached != None:
return cached
split_results = [[] for i in range(7)]
#multi processed forecasting
## WARNING: hangups:
#v.1 : pool.map
#v.2 use multiple processes instead of pool.map to circumvent a hangup caused
# by a multiprocessing/django incompabatility
#share results in a multiaccess queue
#note: this queue can not hold unlimited elements and will hang up with no warning if there are too many elements
#12 weeks 1hourly will work, 20 not
#result_queue = multiprocessing.Queue()
#v.3 now uses a shared multiprocessing dict, to circumvent hang up problems with queue on windows
mgr = multiprocessing.Manager()
dict_threadsafe = mgr.dict()
logger.debug("forecasting demands with daytype strategy..")
#call class as Functor because class methods are not pickeable
jobs = [
Process(target=self, args=(demand, index, dict_threadsafe))
for index, demand in enumerate(self.demands)
]
for job in jobs:
job.start()
for job in jobs:
job.join()
for index in dict_threadsafe.keys():
split_results[index] = dict_threadsafe[index]
forecasted_demands = []
self.calculated_parameters = []
for fc_tuple in split_results:
forecasted_demands.append(list(fc_tuple[0]))
self.calculated_parameters.append(fc_tuple[1])
#cache forecasts
pickle.dump(
{
"forecasts": forecasted_demands,
"parameters": self.calculated_parameters,
"date": self.env.now
},
open(os.path.join(BASE_DIR, "cache", "cached_forecasts.cache"),
"wb"))
logger.debug("forecasting completed")
return forecasted_demands
def run_rule(self, rule, params=None):
try:
log.info('trying job %s' % rule.name)
job_settings = self.settings.copy()
if params:
job_settings.update(params)
name = rule.qualified_name
args = (name, job_settings)
Process(target=run_rule_async, args=args).start()
except Exception:
log.error('Error running rule: %s' % rule.name)
def startNode(self, requests, nodesIds, idx, factory, resultsLogger,
knownHosts):
nodeName = nodesIds[idx]
nodeRequests = requests[nodeName]
client = SimulatorClientProcess(factory.port, nodeName, idx,
resultsLogger, knownHosts)
process = Process(target=client.startNode,
args=(factory, nodeRequests))
self.clients[nodeName] = process
def __init__(self, name=None):
if name is None: name = 'default'
if name in _workers.keys():
return
_workers[name] = self
super(Worker, self).__init__()
self.daemon = True
self.name = name
self._queue = _ver.queue.Queue(1)
self.last_exception = None
self._pon = _mp.Value('b', True)
tsk, self.task = _mp.Pipe(False)
self.out, res = _mp.Pipe(False)
self.process = Process(target=process,
args=(self._pon, tsk, res),
name=name)
self.process.start()
self._on = True
self.start()
_time.sleep(1)
def load(self):
indices_groups = array_split(self.indices, self.procs)
if self.use_lock:
lock = Lock()
else:
lock = None
processes = [
Process(target=self.loader,
args=(indices_group, self.f, self.fkwds, self.Model, lock))
for indices_group in indices_groups
]
self.run(processes)
def setUp(self):
#setup game server to run on a seperate process
self.game_server = Process(target=start_server)
self.game_server.start()
#create the game client
self.client = GameClient(host="127.0.0.1", port="5000")
self.player_one = "Arthur"
self.player_one_suspect = game_state.PEACOCK
self.player_two = "Steven"
self.player_two_suspect = game_state.PLUM
def start(self):
"""
Starts the background process that prunes expired items
"""
def task():
while True:
self.prune_expired()
pytime.sleep(.5)
self.processs = Process(target=task)
self.processs.start()
def test_tornado(self):
def run_provider(queue):
try:
provider = create_provider()
app = TornadoApplication([
url(r"/authorize", OAuth2Handler, dict(provider=provider)),
url(r"/token", OAuth2Handler, dict(provider=provider))
],
debug=True)
app.listen(15486)
queue.put({"result": 0})
IOLoop.current().start()
except Exception as e:
queue.put({"result": 1, "error_message": str(e)})
ready_queue = Queue()
self.server = Process(target=run_provider, args=(ready_queue, ))
self.server.start()
provider_started = ready_queue.get()
if provider_started["result"] != 0:
raise Exception("Error starting Provider process with message"
"'{0}'".format(provider_started["error_message"]))
self.client = Process(target=run_client, args=(ready_queue, ))
self.client.start()
client_started = ready_queue.get()
if client_started["result"] != 0:
raise Exception("Error starting Client Application process with "
"message '{0}'".format(
client_started["error_message"]))
self.access_token()
def test_wsgi(self):
def run_provider(queue):
try:
provider = create_provider()
app = Application(provider=provider)
httpd = make_server('',
15486,
app,
handler_class=NoLoggingHandler)
queue.put({"result": 0})
httpd.serve_forever()
except Exception as e:
queue.put({"result": 1, "error_message": str(e)})
ready_queue = Queue()
self.server = Process(target=run_provider, args=(ready_queue, ))
self.server.start()
provider_started = ready_queue.get()
if provider_started["result"] != 0:
raise Exception("Error starting Provider process with message"
"'{0}'".format(provider_started["error_message"]))
self.client = Process(target=run_client, args=(ready_queue, ))
self.client.start()
client_started = ready_queue.get()
if client_started["result"] != 0:
raise Exception("Error starting Client Application process with "
"message '{0}'".format(
client_started["error_message"]))
self.access_token()
class ChordNode:
process = None
node = None
def createNode(self, nodeId, port, cacheStorage, knownHosts):
BlockingChordNode(nodeId, knownHosts, port, cacheStorage)
def __init__(self, knownHosts, port=4000, cacheStorage=None):
# TODO id?
nodeId = random.randrange(NETWORK_SIZE)
self.process = Process(target=self.createNode,
args=(nodeId, port, cacheStorage, knownHosts))
self.process.start()
self.node = ServerProxy(('', port), nodeId, ('localhost', port))
def terminate(self):
"""
Kill node process
"""
self.process.terminate()
def cli(config):
_configure_logging()
config = yaml.load(config)
for plugin_config in config['plugins']:
plugin_config['redis'] = config['redis']
_launch(plugin_config['type'], plugin_config, False)
sleep(3) # wait for plugins to subscribe to queues
from triviabot.daemon import Daemon
core_config = config['core']
core_config['redis'] = config['redis']
daemon = Daemon(core_config)
p = Process(target=daemon.run)
p.daemon = True
p.run()
while True:
sleep(1)
def __init__(self, settings):
logging.debug("Initializing Router")
self.dbhost = settings['DBHOST']
self.poll = float(settings['POLLINTERVAL'])
self.agent = settings['JOBSURL']
user = settings['RTRUSER']
mapfile = settings['MAPFILE']
key = None
if 'RTRKEY' in settings:
key = settings['RTRKEY']
self.map = self._load_mapfile(mapfile)
self.vyos = vyos_interface.vyosInterface(user, key=key)
self.ddns = self._init_dns(settings)
self.shutdown_file = settings.get('SHUTFILE', "/tmp/shutdown_sdn")
self.collection = settings.get('COLLECTION', 'routes')
self.cleanup_proc = Process(target=self.cleanup, name='CleanupThread')
self.cleanup_proc.start()
client = MongoClient(self.dbhost)
self.routes = client.sdn[self.collection]
if self.routes.find_one() is None:
self.cleanup_proc.terminate()
raise OSError('DB not initialized')
def main():
auth_server = Process(target=run_auth_server)
auth_server.start()
app_server = Process(target=run_app_server)
app_server.start()
print("Access http://localhost:8081/app in your browser")
def sigint_handler(signal, frame):
print("Terminating servers...")
auth_server.terminate()
auth_server.join()
app_server.terminate()
app_server.join()
signal.signal(signal.SIGINT, sigint_handler)
def run(self):
items = get_file_list(self.path_list) #取得所有文件列表，读取文件交给了map_workers
self.file_list = items
self.total_count = len(items)
item_step_count = self.total_count / self.map_worker_count #每个worker分得的任务数量
print u'文件总数：%d' % self.total_count
for i in range(self.map_worker_count):
name = 'pre_worker@%d' % (i + 1)
p = Process(target=add_worker,args=(name,self.task_queue,self.result_queue))
self.worker_process_list.append(p)
self.start_time = time.clock()
self.last_refresh_time = time.clock()
print u'开始时间:',datetime.now()
for p in self.worker_process_list:
p.start()
t_reader = threading.Thread(target=self.put_task_thread)
t_writer = threading.Thread(target=self.get_result_thread)
t_reader.start()
t_writer.start()
print '------------------------'
def get_stream(hdf5_file, which_set, batch_size=None):
dataset = TrajectoryDataset(which_sets=(which_set, ))
if batch_size == None:
batch_size = dataset.num_examples
data_stream = DataStream(dataset=dataset,
iteration_scheme=ShuffledScheme(
examples=dataset.num_examples,
batch_size=batch_size))
load_in_memory = os.path.getsize(
hdf5_file) < 14 * 10**9 or which_set == 'test'
if not load_in_memory:
port = 5557 if which_set == 'train' else 5558
print port
server_process = Process(target=start_server,
args=(data_stream, port, 10))
server_process.start()
data_stream = ServerDataStream(dataset.sources,
False,
host='localhost',
port=port,
hwm=10)
return data_stream
def test_mcdpweb_server(dirname):
port = random.randint(11000, 15000)
base = 'http://127.0.0.1:%s' % port
p = Process(target=start_server, args=(
dirname,
port,
))
p.start()
print('sleeping')
time.sleep(5)
try:
url_wrong = base + '/not-existing'
urllib2.urlopen(url_wrong).read()
except HTTPError:
pass
else:
raise Exception('Expected 404')
# now run the spider
tmpdir = tempfile.mkdtemp(prefix='wget-output')
cwd = '.'
cmd = ['wget', '-nv', '-P', tmpdir, '-m', base]
# res = system_cmd_result(
# cwd, cmd,
# display_stdout=True,
# display_stderr=True,
# raise_on_error=True)
sub = subprocess.Popen(cmd, bufsize=0, cwd=cwd)
sub.wait()
exc = get_exceptions(port)
if len(exc) == 0:
msg = 'Expected at least a not-found error'
raise Exception(msg)
if not 'not-existing' in exc[0]:
raise Exception('Could not find 404 error')
exc = exc[1:]
if exc:
msg = 'Execution raised errors:\n\n'
msg += str("\n---\n".join(exc))
raise_desc(Exception, msg)
url_exit = base + '/exit'
urllib2.urlopen(url_exit).read()
print('waiting for start_server() process to exit...')
p.join()
print('...clean exit')
def _launch(type_name, config, is_important):
plugin_clazz = _get_type(type_name)
if not plugin_clazz:
logging.warn('could not find %s plugin' % type_name)
d = plugin_clazz(config)
p = Process(target=d.start)
p.daemon = not is_important
p.name = 'plugin: %s' % d.name
p.start()
def calculate(self, funcs):
## always count the data
base_funcs = [{'function': len, 'name': 'count_agg', 'raw': False}]
## check if there are raw calculation functions
if any([f['raw'] for f in funcs]):
has_raw = True
if self.sub.value_set == {}:
raise (ValueError(
'Raw aggregate statistics requested with no "value_set_coll"!!'
))
else:
has_raw = False
## need to count the raw data values if raw value are present
if has_raw:
base_funcs.append({
'function': len,
'name': 'count_raw',
'raw': True
})
## append the rest of the functions
funcs = base_funcs + funcs
## check the function definition dictionary for common problems
check_function_dictionary(funcs)
## convert the time vector for faster referencing
time_conv = [[getattr(time, grp) for grp in self.time_grouping]
for time in self.sub.timevec]
## the shared list
all_attrs = Manager().list()
if self.procs > 1:
processes = [
Process(target=self.f_calculate,
args=(all_attrs, self.sub, groups, funcs, time_conv,
self.time_grouping, self.grouping, has_raw))
for groups in self.groups
]
pmanager = ProcessManager(processes, self.procs)
pmanager.run()
else:
self.f_calculate(all_attrs, self.sub, self.groups[0], funcs,
time_conv, self.time_grouping, self.grouping,
has_raw)
self.stats = merge_dict_list(list(all_attrs))
def calculate(self, funcs):
## always count the data
funcs = [{'function': len, 'name': 'count'}] + funcs
## check the function definition dictionary for common problems
check_function_dictionary(funcs)
## convert the time vector for faster referencing
time_conv = [[getattr(time, grp) for grp in self.time_grouping]
for time in self.sub.timevec]
## the shared list
all_attrs = Manager().list()
if self.procs > 1:
processes = [
Process(target=self.f_calculate,
args=(all_attrs, self.sub, groups, funcs, time_conv,
self.time_grouping, self.grouping))
for groups in self.groups
]
self.run_parallel(processes)
else:
self.f_calculate(all_attrs, self.sub, self.groups[0], funcs,
time_conv, self.time_grouping, self.grouping)
return (all_attrs)
def test_cleanup(self):
"""
Test the cleanup thread
"""
# We want the cleanup thread to run in this context
# but it is an infitite loop. So let's delay a shtudown.
sd2 = "/tmp/shut2"
rec = {
'address': '1.2.3.4',
'ip': '10.128.0.1',
'router': 'router',
'last_associated': time(),
'end_time': 0,
'uid': 501,
'user': '******',
'jobid': '1233',
'status': 'used'
}
self.db.routes2.insert(rec)
# We want to shutdown the thread that is started on init
with open(sd2, 'w') as f:
f.write('1')
settings = self.settings.copy()
settings['COLLECTION'] = 'routes2'
settings['SHUTFILE'] = sd2
rt = router.Router(settings)
# Wait for the init thread to shutdown
sleep(0.2)
# Now let's start our own
if os.path.exists(sd2):
os.remove(sd2)
shut = Process(target=self._shutdown)
shut.start()
rt.cleanup()
shut.terminate()
r = self.db.routes2.find_one({'address': '1.2.3.4'})
self.assertEquals(r['status'], 'available')
self.db.routes2.remove({})
rv = rt.cleanup()
self.assertEquals(-1, rv)
def start_example_app_process3():
# XXX DO NOT FORGET TO KILL THE PROCESS IF THE TEST DOES NOT SUCCEED
p = Process(target=start_example_server3)
p.start()
return p
'density_configs': ["sgde_zero"], #density_configs,
"function_configs": [("two_moons", 2), ("mult_beta", 2)]
}
processes = []
for estimationMethod in scenarions['density_configs']:
for functionName, numDims in scenarions["function_configs"]:
print("-" * 80)
print("scenario: %s, d=%i, %s" %
(functionName, numDims, estimationMethod))
print("-" * 80)
if args.parallel:
myargs = (functionName, estimationMethod, 10, numDims, 20000,
"join", "ml", args.out, args.plot)
processes.append(
Process(target=run_densityEstimation, args=myargs))
else:
run_densityEstimation(functionName,
estimationMethod,
kfold=10,
numDims=numDims,
numSamples=20000,
candidates="join",
bandwidthOptimizationType="ml",
out=args.out,
plot=args.plot)
# run applications in parallel if there are any available
for process in processes:
process.start()
def action_import_apks(storage, apk_paths,
copy_apk = False, copy_to_mongodb = False,
update = False, tag = None,
# shared memory
cnt_imported_apks = None, total_apk_count = None, import_finished = None,
# concurrent settings
concurrency = None
):
''' Import the apks from the `apk_paths` and create the file system structure
where the results will be kept, specified by `storage`.
Parameters
----------
storage : RedundantStorage
The store to use.
apk_paths : iterable<str>
The apk files and/or directories.
copy_apk : bool
Import the apk file to the `import_dir` (copy it).
copy_to_mongodb : bool, optional (default is False)
Also import into MongoDB. Useful for the distributed analysis.
update : bool
Update apks that have already been imported.
tag : str, optional (default is None)
Some tag
cnt_imported_apks : multiprocessing.Value<int>, optional (default is None)
If given, use for progress updating.
total_apk_count : multiprocessing.Value<int>, optional (default is None)
If given, use for total count of apks.
import_finished : multiprocessing.Value<byte>, optional (default is None)
If given, use to signal that import has been completed.
concurrency : int, optional (default is number of cpus)
Number of processes to use for the import.
'''
from androlyze.loader.ApkImporter import ApkImporter
# get single paths to apks so we get the correct total count of apks
clilog.info("looking for apks in given paths ... ")
apk_paths = ApkImporter.get_apks_from_list_or_dir(apk_paths)
if total_apk_count is not None:
# may be time consuming for recursive lookup
apk_paths, total_apk_count.value = Util.count_iterable_n_clone(apk_paths)
# create count if not given
if cnt_imported_apks is None:
cnt_imported_apks = Value('i', 0, lock = RLock())
# set concurrency
if concurrency is None:
concurrency = cpu_count()
log.warn("Using %d processes", concurrency)
clilog.info("Storage dir is %s" % storage.fs_storage.store_root_dir)
if copy_apk:
clilog.info("Copying APKs to %s ..." % storage.fs_storage.store_root_dir)
def import_apks(apk_paths):
apk_importer = ApkImporter(apk_paths, storage)
for apk in apk_importer.import_apks(copy_apk = copy_apk, copy_to_mongodb = copy_to_mongodb,
update = update, tag = tag):
clilog.info("imported %s", apk.short_description())
# use shared memory counter if given
if cnt_imported_apks is not None:
with cnt_imported_apks.get_lock():
cnt_imported_apks.value += 1
pool = []
# don't convert generator to list if only 1 process wanted
apk_paths = [apk_paths] if concurrency == 1 else Util.split_n_uniform_distri(list(apk_paths), concurrency)
# start parallel import
# multiprocessing's pool causes pickle errors
for i in range(concurrency):
p = Process(target = import_apks, args = (apk_paths[i], ))
log.debug("starting process %s", p)
pool.append(p)
p.start()
for it in pool:
log.debug("joined on process %s", p)
it.join()
apks_imported = cnt_imported_apks.value != 0
# show some message that no APK has been imported
if not apks_imported:
log.warn("No .apk file has been imported! This means no .apk file has been found or they already have been imported.")
else:
clilog.info("done")
# because not all apks may be importable, we cannot use we count for signal that the import is done
if import_finished is not None:
import_finished.value = 1
clilog.info("Imported %d apks", cnt_imported_apks.value)
class ImageMngr(object):
"""
This class handles most of the backend work for the image gateway.
It uses a Mongo Database to track state, uses threads to dispatch work,
and has public functions to lookup, pull and expire images.
"""
def __init__(self, config, logger=None, logname='imagemngr'):
"""
Create an instance of the image manager.
"""
if logger is None and logname is None:
self.logger = logging.getLogger(logname)
log_handler = logging.StreamHandler()
logfmt = '%(asctime)s [%(name)s] %(levelname)s : %(message)s'
log_handler.setFormatter(logging.Formatter(logfmt))
log_handler.setLevel(logging.INFO)
self.logger.addHandler(log_handler)
elif logname is not None:
self.logger = logging.getLogger(logname)
self.logger.info('ImageMngr using logname %s' % (logname))
else:
print "Using upstream logger"
self.logger = logger
print logger
self.logger.info('ImageMngr using upstream logger')
self.logger.debug('Initializing image manager')
self.config = config
if 'Platforms' not in self.config:
raise NameError('Platforms not defined')
self.systems = []
# Time before another pull can be attempted
self.pullupdatetimeout = 300
if 'PullUpdateTime' in self.config:
self.pullupdatetimeout = self.config['PullUpdateTimeout']
# Max amount of time to allow for a pull
self.pulltimeout = self.pullupdatetimeout
# This is not intended to provide security, but just
# provide a basic check that a session object is correct
self.magic = 'imagemngrmagic'
if 'Authentication' not in self.config:
self.config['Authentication'] = "munge"
self.auth = Authentication(self.config)
self.platforms = self.config['Platforms']
for system in self.config['Platforms']:
self.systems.append(system)
# Connect to database
if 'MongoDBURI' not in self.config:
raise NameError('MongoDBURI not defined')
self.workers = WorkerThreads()
self.status_queue = self.workers.get_updater_queue()
self.status_proc = Process(target=self.status_thread,
name='StatusThread')
self.status_proc.start()
atexit.register(self.shutdown)
self.mongo_init()
def shutdown(self):
self.status_queue.put('stop')
def mongo_init(self):
client = MongoClient(self.config['MongoDBURI'])
db_ = self.config['MongoDB']
self.images = client[db_].images
self.metrics = None
if 'Metrics' in self.config and self.config['Metrics'] is True:
self.metrics = client[db_].metrics
def status_thread(self):
"""
This listens for update messages from a queue.
"""
self.mongo_init()
while True:
message = self.status_queue.get()
if message == 'stop':
self.logger.info("Shutting down Status Thread")
break
ident = message['id']
state = message['state']
meta = message['meta']
# TODO: Handle a failed expire
if state == "FAILURE":
self.logger.warn("Operation failed for %s", ident)
# print "Status: %s" % (state)
# A response message
if state != 'READY':
self.update_mongo_state(ident, state, meta)
continue
if 'response' in meta and meta['response']:
response = meta['response']
self.logger.debug(response)
if 'meta_only' in response:
self.logger.debug('Updating ACLs')
self.update_acls(ident, response)
else:
self.complete_pull(ident, response)
self.logger.debug('meta=%s', str(response))
def check_session(self, session, system=None):
"""Check if this is a valid session
session is a session handle
"""
if 'magic' not in session:
self.logger.warn("request recieved with no magic")
return False
elif session['magic'] is not self.magic:
self.logger.warn("request received with bad magic %s",
session['magic'])
return False
if system is not None and session['system'] != system:
self.logger.warn("request received with a bad system %s!=%s",
session['system'], system)
return False
return True
def _isadmin(self, session, system=None):
"""
Check if this is an admin user.
Returns true if is an admin or false if not.
"""
if 'admins' not in self.platforms[system]:
return False
admins = self.platforms[system]['admins']
user = session['user']
if user in admins:
self.logger.info('user %s is an admin', user)
return True
return False
def _isasystem(self, system):
"""Check if system is a valid platform."""
return bool(system in self.systems)
def _get_groups(self, uid, gid):
"""Look up auxilary groups. """
proc = Popen(['id', '-G', '%d' % (uid)], stdout=PIPE, stderr=PIPE)
if proc is None:
self.logger.warn("Group lookup failed")
return []
stdout, stderr = proc.communicate()
groups = []
for group in stdout.split():
groups.append(int(group))
return groups
def _checkread(self, session, rec):
"""
Checks if the user has read permissions to the image.
"""
# Start by checking if the image is public (no ACLs)
if 'private' in rec and rec['private'] is False:
return True
iUACL = None
iGACL = None
if 'userACL' in rec:
iUACL = rec['userACL']
if 'groupACL' in rec:
iGACL = rec['groupACL']
if iUACL is None and iGACL is None:
return True
if iUACL == [] and iGACL == []:
return True
uid = session['uid']
gid = session['gid']
self.logger.debug('uid=%s iUACL=%s' % (uid, str(iUACL)))
self.logger.debug('sessions = ' + str(session))
groups = self._get_groups(uid, gid)
if iUACL is not None and uid in iUACL:
return True
if iGACL is not None and gid in iGACL:
return True
for group in groups:
if iGACL is not None and group in iGACL:
return True
return False
def _resetexpire(self, ident):
"""Reset the expire time. (Not fully implemented)."""
# Change expire time for image
# TODO shore up expire-time parsing
expire_timeout = self.config['ImageExpirationTimeout']
(days, hours, minutes, secs) = expire_timeout.split(':')
expire = time() + int(secs) + 60 * (int(minutes) + 60 *
(int(hours) + 24 * int(days)))
self._images_update({'_id': ident}, {'$set': {'expiration': expire}})
return expire
def _make_acl(self, acllist, id):
if id not in acllist:
acllist.append(id)
return acllist
def _compare_list(self, a, b, key):
""""
look at the key element of two objects
and compare the list of ids.
return True if everything matches
return False if anything is different
"""
# If the key isn't in the objects or
# something else fails, then it must
# have changed.
try:
if key not in a:
return False
if key not in b:
return False
except:
return True
aitems = a[key]
bitems = b[key]
if len(aitems) != len(bitems):
return False
for item in aitems:
if item not in bitems:
return False
return True
def _add_metrics(self, session, request, record):
"""
Add a row to mongo for this lookup request.
"""
try:
r = {
'user': session['user'],
'uid': session['uid'],
'system': request['system'],
'type': request['itype'],
'tag': request['tag'],
'id': record['id'],
'time': time()
}
self._metrics_insert(r)
except:
self.logger.warn('Failed to log lookup.')
def get_metrics(self, session, system, limit):
"""
Return the last <limit> lookup records.
"""
recs = []
if not self._isadmin(session, system):
return recs
if self.metrics is None:
return recs
count = self.metrics.count()
skip = count - limit
if skip < 0:
skip = 0
for r in self.metrics.find().skip(skip):
r.pop('_id', None)
recs.append(r)
return recs
def new_session(self, auth_string, system):
"""
Creates a session context that can be used for multiple transactions.
auth is an auth string that will be passed to the authenication layer.
Returns a context that can be used for subsequent operations.
"""
if auth_string is None:
return {'magic': self.magic, 'system': system}
arec = self.auth.authenticate(auth_string, system)
if arec is None and isinstance(arec, dict):
raise OSError("Authenication returned None")
else:
if 'user' not in arec:
raise OSError("Authentication returned invalid response")
session = arec
session['magic'] = self.magic
session['system'] = system
return session
def lookup(self, session, image):
"""
Lookup an image.
Image is dictionary with system,itype and tag defined.
"""
if not self.check_session(session, image['system']):
raise OSError("Invalid Session")
query = {
'status': 'READY',
'system': image['system'],
'itype': image['itype'],
'tag': {
'$in': [image['tag']]
}
}
self.update_states()
rec = self._images_find_one(query)
if rec is not None:
if self._checkread(session, rec) is False:
return None
self._resetexpire(rec['_id'])
if self.metrics is not None:
self._add_metrics(session, image, rec)
return rec
def imglist(self, session, system):
"""
list images for a system.
Image is dictionary with system defined.
"""
if not self.check_session(session, system):
raise OSError("Invalid Session")
if self._isasystem(system) is False:
raise OSError("Invalid System")
query = {'status': 'READY', 'system': system}
self.update_states()
records = self._images_find(query)
resp = []
for record in records:
if self._checkread(session, record):
resp.append(record)
# verify access
return resp
def show_queue(self, session, system):
"""
list queue for a system.
Image is dictionary with system defined.
"""
if not self.check_session(session, system):
raise OSError("Invalid Session")
query = {'status': {'$ne': 'READY'}, 'system': system}
self.update_states()
records = self._images_find(query)
resp = []
for record in records:
resp.append({
'status': record['status'],
'image': record['pulltag']
})
return resp
def _isready(self, image):
"""Helper function to determine if an image is READY."""
query = {
'status': 'READY',
'system': image['system'],
'itype': image['itype'],
'tag': {
'$in': [image['tag']]
}
}
rec = self._images_find_one(query)
if rec is not None:
return True
return False
def _pullable(self, rec):
"""
An image is pullable when:
-There is no existing record
-The status is a FAILURE
-The status is READY and it is past the update time
-The state is something else and the pull has expired
"""
# if rec is None then do a pull
if rec is None:
return True
# Okay there has been a pull before
# If the status flag is missing just repull (shouldn't happen)
if 'status' not in rec:
return True
status = rec['status']
# EXPIRED images can be pulled
if status == 'EXPIRED':
return True
# Need to deal with last_pull for a READY record
if 'last_pull' not in rec:
return True
nextpull = self.pullupdatetimeout + rec['last_pull']
# It has been a while, so re-pull to see if it is fresh
if status == 'READY' and (time() > nextpull):
return True
# Repull failed pulls
if status == 'FAILURE' and (time() > nextpull):
return True
# Last thing... What if the pull somehow got hung or died in the middle
# See if heartbeat is old
# TODO: add pull timeout. For now use 1 hour
if status != 'READY' and 'last_heartbeat' in rec:
if (time() - rec['last_heartbeat']) > 3600:
return True
return False
def new_pull_record(self, image):
"""
Creates a new image in mongo. If the pull already exist it removes
it first.
"""
# Clean out any existing records
for rec in self._images_find(image):
if rec['status'] == 'READY':
continue
else:
self._images_remove({'_id': rec['_id']})
newimage = {
'format': 'invalid', # <ext4|squashfs|vfs>
'arch': 'amd64', # <amd64|...>
'os': 'linux', # <linux|...>
'location': '', # urlencoded location
'remotetype': 'dockerv2', # <file|dockerv2|amazonec2>
'ostcount': '0', # integer, number of OSTs (future)
'replication': '1', # integer, number of copies to deploy
'userACL': [],
'groupACL': [],
'private': None,
'tag': [],
'status': 'INIT'
}
if 'DefaultImageFormat' in self.config:
newimage['format'] = self.config['DefaultImageFormat']
for param in image:
if param is 'tag':
continue
newimage[param] = image[param]
self._images_insert(newimage)
return newimage
def pull(self, session, image, testmode=0):
"""
pull the image
Takes an auth token, a request object
Optional: testmode={0,1,2} See below...
"""
request = {
'system': image['system'],
'itype': image['itype'],
'pulltag': image['tag']
}
self.logger.debug('Pull called Test Mode=%d', testmode)
if not self.check_session(session, request['system']):
self.logger.warn('Invalid session on system %s', request['system'])
raise OSError("Invalid Session")
# If a pull request exist for this tag
# check to see if it is expired or a failure, if so remove it
# otherwise
# return the record
rec = None
# find any pull record
self.update_states()
# let's lookup the active image
query = {
'status': 'READY',
'system': image['system'],
'itype': image['itype'],
'tag': {
'$in': [image['tag']]
}
}
rec = self._images_find_one(query)
for record in self._images_find(request):
status = record['status']
if status == 'READY' or status == 'SUCCESS':
continue
rec = record
break
inflight = False
recent = False
if rec is not None and rec['status'] != 'READY':
inflight = True
elif rec is not None:
# if an image has been pulled in the last 60 seconds
# let's consider that "recent"
if (time() - rec['last_pull']) < 10:
recent = True
request['userACL'] = []
request['groupACL'] = []
if 'userACL' in image and image['userACL'] != []:
request['userACL'] = self._make_acl(image['userACL'],
session['uid'])
if 'groupACL' in image and image['groupACL'] != []:
request['groupACL'] = self._make_acl(image['groupACL'],
session['gid'])
if self._compare_list(request, rec, 'userACL') and \
self._compare_list(request, rec, 'groupACL'):
acl_changed = False
else:
self.logger.debug("No ACL change detected.")
acl_changed = True
# We could hit a key error or some other edge case
# so just do our best and update if there are problems
update = False
if not recent and not inflight and acl_changed:
self.logger.debug("ACL change detected.")
update = True
if self._pullable(rec):
self.logger.debug("Pullable image")
update = True
if update:
self.logger.debug("Creating New Pull Record")
rec = self.new_pull_record(request)
ident = rec['_id']
self.logger.debug("ENQUEUEING Request")
self.update_mongo_state(ident, 'ENQUEUED')
request['tag'] = request['pulltag']
request['session'] = session
self.logger.debug("Calling do pull with queue=%s",
request['system'])
self.workers.dopull(ident, request, testmode=testmode)
memo = "pull request queued s=%s t=%s" \
% (request['system'], request['tag'])
self.logger.info(memo)
self.update_mongo(ident, {'last_pull': time()})
return rec
def mngrimport(self, session, image, testmode=0):
"""
import the image directly from a file
Only for allowed users
Takes an auth token, a request object
"""
meta = {}
fp = image['filepath']
request = {
'system': image['system'],
'itype': image['itype'],
'pulltag': image['tag'],
'filepath': image['filepath'],
'format': image['format'],
'meta': meta
}
self.logger.debug('mngrmport called for file %s' % (fp))
# self.logger.debug(image)
if not self.check_session(session, request['system']):
self.logger.warn('Invalid session on system %s', request['system'])
raise OSError("Invalid Session")
# Skip checks about previous requests for now
# Future work could check the fasthash and
# not import if they're the same
q = {
'system': image['system'],
'itype': image['itype'],
'pulltag': image['tag']
}
rec = self._images_find_one(q)
if not self._pullable(rec):
return rec
# rec = None
# request['userACL'] = []
# request['groupACL'] = []
# if 'userACL' in image and image['userACL'] != []:
# request['userACL'] = self._make_acl(image['userACL'],
# session['uid'])
# if 'groupACL' in image and image['groupACL'] != []:
# request['groupACL'] = self._make_acl(image['groupACL'],
# session['gid'])
# if self._compare_list(request, rec, 'userACL') and \
# self._compare_list(request, rec, 'groupACL'):
# acl_changed = False
# else:
# self.logger.debug("No ACL change detected.")
# acl_changed = True
# We could hit a key error or some other edge case
# so just do our best and update if there are problems
self.logger.debug("Creating New Import Record")
# new_pull_record works for import too
rec = self.new_pull_record(request)
ident = rec['_id']
self.logger.debug("ENQUEUEING Request, ident %s" % (ident))
self.update_mongo_state(ident, 'ENQUEUED')
request['tag'] = request['pulltag']
request['session'] = session
self.logger.debug("Calling wrkimport with queue=%s", request['system'])
self.workers.dowrkimport(ident, request, testmode=testmode)
memo = "import request queued s=%s t=%s" \
% (request['system'], request['tag'])
self.logger.info(memo)
self.update_mongo(ident, {'last_pull': time()})
return rec
def update_mongo_state(self, ident, state, info=None):
"""
Helper function to set the mongo state for an image with _id==ident
to state=state.
"""
if state == 'SUCCESS':
state = 'READY'
set_list = {'status': state, 'status_message': ''}
if info is not None and isinstance(info, dict):
if 'heartbeat' in info:
set_list['last_heartbeat'] = info['heartbeat']
if 'message' in info:
set_list['status_message'] = info['message']
self._images_update({'_id': ident}, {'$set': set_list})
def add_tag(self, ident, system, tag):
"""
Helper function to add a tag to an image.
ident is the mongo id (not image id)
"""
# Remove the tag first
self.remove_tag(system, tag)
# see if tag isn't a list
rec = self._images_find_one({'_id': ident})
if rec is not None and 'tag' in rec and \
not isinstance(rec['tag'], (list)):
memo = 'Fixing tag for non-list %s %s' % (ident, str(rec['tag']))
self.logger.info(memo)
curtag = rec['tag']
self._images_update({'_id': ident}, {'$set': {'tag': [curtag]}})
self._images_update({'_id': ident}, {'$addToSet': {'tag': tag}})
return True
def remove_tag(self, system, tag):
"""
Helper function to remove a tag to an image.
"""
self._images_update({
'system': system,
'tag': {
'$in': [tag]
}
}, {'$pull': {
'tag': tag
}},
multi=True)
return True
def update_acls(self, ident, response):
self.logger.debug("Update ACLs called for %s %s", ident, str(response))
pullrec = self._images_find_one({'_id': ident})
if pullrec is None:
self.logger.error('ERROR: Missing pull request (r=%s)',
str(response))
return
# Check that this image ident doesn't already exist for this system
rec = self._images_find_one({
'id': response['id'],
'status': 'READY',
'system': pullrec['system']
})
if rec is None:
# This means the image already existed, but we didn't have a
# record of it. That seems odd (it happens in tests). Let's
# note it and power on through.
msg = "WARNING: No image record found for an ACL update"
self.logger.warn(msg)
response['last_pull'] = time()
response['status'] = 'READY'
self.update_mongo(ident, response)
self.add_tag(ident, pullrec['system'], pullrec['pulltag'])
else:
updates = {
'userACL': response['userACL'],
'groupACL': response['groupACL'],
'private': response['private'],
'last_pull': time()
}
self.logger.debug("Doing ACLs update")
response['last_pull'] = time()
response['status'] = 'READY'
self.update_mongo(rec['_id'], updates)
self._images_remove({'_id': ident})
def complete_pull(self, ident, response):
"""
Transition a completed pull request to an available image.
"""
self.logger.debug("Complete called for %s %s", ident, str(response))
pullrec = self._images_find_one({'_id': ident})
if pullrec is None:
self.logger.warn('Missing pull request (r=%s)', str(response))
return
# Check that this image ident doesn't already exist for this system
rec = self._images_find_one({
'id': response['id'],
'system': pullrec['system'],
'status': 'READY'
})
tag = pullrec['pulltag']
if rec is not None:
# So we already had this image.
# Let's delete the pull record.
# TODO: update the pull time of the matching id
self.logger.warn('Duplicate image')
update_rec = {'last_pull': time()}
self.update_mongo(rec['_id'], update_rec)
self._images_remove({'_id': ident})
# However it could be a new tag. So let's update the tag
try:
rec['tag'].index(response['tag'])
except:
self.add_tag(rec['_id'], pullrec['system'], tag)
return True
else:
response['last_pull'] = time()
response['status'] = 'READY'
self.update_mongo(ident, response)
self.add_tag(ident, pullrec['system'], tag)
def update_mongo(self, ident, resp):
"""
Helper function to set the mongo values for an image with _id==ident.
"""
setline = dict()
# This maps from the key name in the response to the
# key name used in mongo
mappings = {
'id': 'id',
'entrypoint': 'ENTRY',
'env': 'ENV',
'workdir': 'WORKDIR',
'last_pull': 'last_pull',
'userACL': 'userACL',
'groupACL': 'groupACL',
'private': 'private',
'status': 'status'
}
if 'private' in resp and resp['private'] is False:
resp['userACL'] = []
resp['groupACL'] = []
for key in mappings.keys():
if key in resp:
setline[mappings[key]] = resp[key]
self._images_update({'_id': ident}, {'$set': setline})
def get_state(self, ident):
"""
Lookup the state of the image with _id==ident in Mongo.
Returns the state.
"""
self.update_states()
rec = self._images_find_one({'_id': ident}, {'status': 1})
if rec is None:
return None
elif 'status' not in rec:
return None
return rec['status']
def update_states(self):
"""
Cleanup failed transcations after a period
"""
for rec in self._images_find({'status': 'FAILURE'}):
nextpull = self.pullupdatetimeout + rec['last_pull']
# It it has been a while then let's clean up
if time() > nextpull:
self._images_remove({'_id': rec['_id']})
def autoexpire(self, session, system, testmode=0):
"""Auto expire images and do cleanup"""
# While this should be safe, let's restrict this to admins
if not self._isadmin(session, system):
return False
# Cleanup - Lookup for things stuck in non-READY state
self.update_states()
removed = []
for rec in self._images_find({
'status': {
'$ne': 'READY'
},
'system': system
}):
if 'last_pull' not in rec:
self.logger.warning('Image missing last_pull for pulltag:' +
rec['pulltag'])
continue
if time() > rec['last_pull'] + self.pulltimeout:
removed.append(rec['_id'])
self._images_remove({'_id': rec['_id']})
expired = []
# Look for READY images that haven't been pulled recently
for rec in self._images_find({'status': 'READY', 'system': system}):
if 'expiration' not in rec:
continue
elif rec['expiration'] < time():
self.logger.debug("expiring %s", rec['id'])
ident = rec.pop('_id')
self.expire_id(rec, ident)
if 'id' in rec:
expired.append(rec['id'])
else:
expired.append('unknown')
self.logger.debug(rec['expiration'] > time())
return expired
def expire_id(self, rec, ident, testmode=0):
""" Helper function to expire by id """
memo = "Calling do expire id=%s TM=%d" \
% (ident, testmode)
self.logger.debug(memo)
self.workers.doexpire(ident, rec)
self.logger.info("expire request queued s=%s t=%s", rec['system'],
ident)
def expire(self, session, image, testmode=0):
"""Expire an image. (Not Implemented)"""
if not self._isadmin(session, image['system']):
return False
query = {
'system': image['system'],
'itype': image['itype'],
'tag': {
'$in': [image['tag']]
}
}
rec = self._images_find_one(query)
if rec is None:
return None
ident = rec.pop('_id')
memo = "Calling do expire with queue=%s id=%s TM=%d" \
% (image['system'], ident, testmode)
self.logger.debug(memo)
self.workers.doexpire(ident, rec)
memo = "expire request queued s=%s t=%s" \
% (image['system'], image['tag'])
self.logger.info(memo)
return True
@mongo_reconnect_reattempt
def _images_remove(self, *args, **kwargs):
""" Decorated function to remove images from mongo """
return self.images.remove(*args, **kwargs)
@mongo_reconnect_reattempt
def _images_update(self, *args, **kwargs):
""" Decorated function to updates images in mongo """
return self.images.update(*args, **kwargs)
@mongo_reconnect_reattempt
def _images_find(self, *args, **kwargs):
""" Decorated function to find images in mongo """
return self.images.find(*args, **kwargs)
@mongo_reconnect_reattempt
def _images_find_one(self, *args, **kwargs):
""" Decorated function to find one image in mongo """
return self.images.find_one(*args, **kwargs)
@mongo_reconnect_reattempt
def _images_insert(self, *args, **kwargs):
""" Decorated function to insert an image in mongo """
return self.images.insert(*args, **kwargs)
@mongo_reconnect_reattempt
def _metrics_insert(self, *args, **kwargs):
""" Decorated function to insert an image in mongo """
if self.metrics is not None:
return self.metrics.insert(*args, **kwargs)
toBeAdded.append(server)
if len(toBeAdded) == how_many:
break
for server in toBeAdded:
rest.add_node('Administrator', 'password', server.ip)
#check if its added ?
nodes = rest.node_statuses()
otpNodes = [node.id for node in nodes]
started = rest.rebalance(otpNodes, [])
msg = "rebalance operation started ? {0}"
self.log.info(msg.format(started))
if started:
result = rest.monitorRebalance()
msg = "successfully rebalanced out selected nodes from the cluster ? {0}"
self.log.info(msg.format(result))
return result
return False
if __name__ == "__main__":
process1 = Process(target=start_load, args=(sys.argv,))
process1.start()
process2 = Process(target=start_combo, args=(sys.argv,))
process2.start()
process3 = Process(target=start_backup, args=(sys.argv,))
process3.start()
process1.join()
process2.join()
process3.join()