class TestProject(unittest.TestCase):
def setUp(self):
self.options = Options()
def test_greeting(self):
self.options.parse()
self.assertEquals(self.options.known.name, 'World')
def start(args):
options = Options()
options.parse(args)
project = Project(options)
project.run()
def init_listeners(self):
args = Options().args
watch_hang = args.no_output_timeout >= 0 and \
not args.gdb and \
not args.gdbserver and \
not args.lldb and \
not args.valgrind
watch_fail = not Options().args.is_force
log_output_watcher = LogOutputWatcher()
self.statistics = StatisticsWatcher(log_output_watcher.get_logfile)
self.artifacts = ArtifactsWatcher(log_output_watcher.get_logfile)
output_watcher = OutputWatcher()
self.listeners = [
self.statistics, log_output_watcher, output_watcher, self.artifacts
]
if sampler.is_enabled:
self.listeners.append(sampler.watcher)
if watch_fail:
self.fail_watcher = FailWatcher(self.terminate_all_workers)
self.listeners.append(self.fail_watcher)
if watch_hang:
warn_timeout = 60.0 if args.long else 10.0
hang_watcher = HangWatcher(output_watcher.not_done_worker_ids,
self.kill_all_workers, warn_timeout,
float(args.no_output_timeout))
self.listeners.append(hang_watcher)
def run_project(args):
options = Options()
options.parse(args[1:])
project = Project(options)
print 'Archirtecture chosen:', project.print_arg()
def run_project(args):
options = Options()
options.parse(args[1:])
project = Project(options)
print 'Printing date:', project.date()
print 'Printing example arg:', project.print_example_arg()
def run_project(args):
options = Options()
options.parse(args[1:])
project = Project(options)
print 'Printing date:', project.date()
print 'Printing example arg:', project.print_example_arg()
def run(args):
options = Options()
options.parse(args[1:])
project = Project(options)
print 'Architecture chosen:', project.print_arg()
project.run_command()
def find_suites():
suite_names = Options().args.suites
if suite_names == []:
for root, dirs, names in os.walk(os.getcwd(), followlinks=True):
if "suite.ini" in names:
suite_names.append(os.path.basename(root))
suites = [
TestSuite(suite_name,
Options().args) for suite_name in sorted(suite_names)
]
return suites
class TestCommandLineParameters(unittest.TestCase):
def setUp(self):
self.options = Options()
def test_defaults_options_are_set(self):
opts = self.options.parse()
self.assertEquals(opts.example, 'scramble')
def test_options_example_is_set(self):
opts = self.options.parse(['-x', 'foobar'])
self.assertEquals(opts.example, 'foobar')
opts = self.options.parse(['--example', 'xyz'])
self.assertEquals(opts.example, 'xyz')
class TestCommandLineParameters(unittest.TestCase):
def setUp(self):
self.options = Options()
def test_defaults_options_are_set(self):
opts = self.options.parse()
self.assertEquals(opts.example, 'scramble')
def test_options_example_is_set(self):
opts = self.options.parse(['-x', 'foobar'])
self.assertEquals(opts.example, 'foobar')
opts = self.options.parse(['--example', 'xyz'])
self.assertEquals(opts.example, 'xyz')
def sendMail(msg, type):
MAIL_OPTIONS = Options.OptionReader('MailOptions.txt')
if type == 'alert':
msg['To'] = email.utils.formataddr(('', MAIL_OPTIONS.getValue('alertmail_recipients')))
elif type == 'publication':
msg['To'] = email.utils.formataddr(('', MAIL_OPTIONS.getValue('publicationmail_recipients')))
else:
return
msg['From'] = email.utils.formataddr((MAIL_OPTIONS.getValue('mail_from'), MAIL_OPTIONS.getValue('mail_sender')))
if type == 'alert':
recipients = MAIL_OPTIONS.getValue('alertmail_recipients').split(',')
elif type == 'publication':
recipients = MAIL_OPTIONS.getValue('publicationmail_recipients').split(',')
else:
return
with open(MAIL_OPTIONS.getValue('mail_credentials'), 'r') as f:
password = f.read()[:-1]
server_connection = smtplib.SMTP(MAIL_OPTIONS.getValue('mail_server'), int(MAIL_OPTIONS.getValue('mail_port')))
try:
# initiate connection
server_connection.ehlo()
# Try to encrypt the session
if server_connection.has_extn('STARTTLS'):
server_connection.starttls()
# reinitiate server connection over TLS
server_connection.ehlo()
server_connection.login(MAIL_OPTIONS.getValue('mail_user'), password)
server_connection.sendmail(MAIL_OPTIONS.getValue('mail_sender'), recipients, msg.as_string())
finally:
server_connection.quit()
def process_result(self, obj):
if isinstance(obj, WorkerDone):
bufferized = self.buffer.get(obj.worker_id, '')
if bufferized:
OutputWatcher._write(bufferized, obj.worker_id)
if obj.worker_id in self.buffer.keys():
del self.buffer[obj.worker_id]
return
# Skip irrelevant events.
if not isinstance(obj, WorkerOutput):
return
# Skip color_log() events if --debug is not passed.
if obj.log_only and not Options().args.debug:
return
# Prepend color_log() messages with a timestamp.
if obj.log_only:
prefix = '[{}] '.format(obj.timestamp)
obj.output = prefix_each_line(prefix, obj.output)
bufferized = self.buffer.get(obj.worker_id, '')
if decolor(obj.output).endswith('\n'):
OutputWatcher._write(bufferized + obj.output, obj.worker_id)
self.buffer[obj.worker_id] = ''
else:
self.buffer[obj.worker_id] = bufferized + obj.output
def __init__(self):
self.fds = dict()
self.logdir = os.path.join(Options().args.vardir, 'log')
try:
os.makedirs(self.logdir)
except OSError:
pass
def save_artifacts(self):
if not self.failed_workers:
return
vardir = Options().args.vardir
artifacts_dir = os.path.join(vardir, 'artifacts')
artifacts_log_dir = os.path.join(artifacts_dir, 'log')
artifacts_reproduce_dir = os.path.join(artifacts_dir, 'reproduce')
safe_makedirs(artifacts_dir)
safe_makedirs(artifacts_log_dir)
safe_makedirs(artifacts_reproduce_dir)
for worker_name in self.failed_workers:
logfile = self.get_logfile(worker_name)
reproduce_file_path = get_reproduce_file(worker_name)
shutil.copy(
logfile,
os.path.join(artifacts_log_dir, os.path.basename(logfile)))
shutil.copy(
reproduce_file_path,
os.path.join(artifacts_reproduce_dir,
os.path.basename(reproduce_file_path)))
shutil.copytree(os.path.join(vardir, worker_name),
os.path.join(artifacts_dir, worker_name),
ignore=shutil.ignore_patterns(
'*.socket-iproto', '*.socket-admin', '*.sock',
'*.control'))
shutil.copytree(os.path.join(vardir, 'statistics'),
os.path.join(artifacts_dir, 'statistics'))
def print_statistics(self):
"""Print statistics and results of testing."""
# Prepare standalone subpath '<vardir>/statistics' for statistics files.
stats_dir = os.path.join(Options().args.vardir, 'statistics')
safe_makedirs(stats_dir)
self.print_rss_summary(stats_dir)
self.print_duration(stats_dir)
if self.stats:
color_stdout('Statistics:\n', schema='test_var')
for short_status, cnt in self.stats.items():
color_stdout('* %s: %d\n' % (short_status, cnt), schema='test_var')
if not self.failed_tasks:
return False
color_stdout('Failed tasks:\n', schema='test_var')
for task_id, worker_name, result_checksum, show_reproduce_content in self.failed_tasks:
logfile = self.get_logfile(worker_name)
task_id_str = yaml.safe_dump(task_id, default_flow_style=True)
color_stdout('- %s' % task_id_str, schema='test_var')
color_stdout('# results file checksum: %s\n' % result_checksum)
color_stdout('# logfile: %s\n' % logfile)
reproduce_file_path = get_reproduce_file(worker_name)
color_stdout('# reproduce file: %s\n' % reproduce_file_path)
if show_reproduce_content:
color_stdout("---\n", schema='separator')
print_tail_n(reproduce_file_path)
color_stdout("...\n", schema='separator')
return True
def reproduce_task_groups(task_groups):
"""Filter provided task_groups down to the one certain group. Sort tests in
this group as in the reproduce file.
"""
found_keys = []
reproduce = parse_reproduce_file(Options().args.reproduce)
if not reproduce:
raise ValueError('[reproduce] Tests list cannot be empty')
for i, task_id in enumerate(reproduce):
for key, task_group in task_groups.items():
if task_id in task_group['task_ids']:
found_keys.append(key.rstrip('_fragile'))
break
if len(found_keys) != i + 1:
raise ValueError('[reproduce] Cannot find test "%s"' %
str(task_id))
found_keys = list(set(found_keys))
if len(found_keys) < 1:
raise ValueError('[reproduce] Cannot find any suite for given tests')
elif len(found_keys) > 1:
raise ValueError(
'[reproduce] Given tests contained by different suites')
res_key = found_keys[0]
res_task_group = copy.deepcopy(task_groups[key])
res_task_group['task_ids'] = reproduce
return {res_key: res_task_group}
def run_test(self, test, server, inspector):
""" Returns short status of the test as a string: 'skip', 'pass',
'new', 'fail', or 'disabled' and results file checksum on fail.
"""
test.inspector = inspector
test_name = os.path.basename(test.name)
full_test_name = os.path.join(self.ini['suite'], test_name)
color_stdout(just_and_trim(full_test_name, 47) + ' ', schema='t_name')
# for better diagnostics in case of a long-running test
conf = ''
if test.run_params:
conf = test.conf_name
color_stdout(just_and_trim(conf, 15) + ' ', schema='test_var')
start_time = time.time()
if self.is_test_enabled(test, conf, server):
short_status, result_checksum = test.run(server)
else:
color_stdout("[ disabled ]\n", schema='t_name')
short_status = 'disabled'
result_checksum = None
duration = time.time() - start_time
# cleanup only if test passed or if --force mode enabled
if Options().args.is_force or short_status == 'pass':
inspector.cleanup_nondefault()
return short_status, result_checksum, duration
def collect_tests(self):
if self.tests_are_collected:
return self.tests
if self.ini['core'] == 'tarantool':
TarantoolServer.find_tests(self, self.suite_path)
elif self.ini['core'] == 'app':
AppServer.find_tests(self, self.suite_path)
elif self.ini['core'] == 'unittest':
UnittestServer.find_tests(self, self.suite_path)
elif self.ini['core'] == 'stress':
# parallel tests are not supported and disabled for now
self.tests = []
self.tests_are_collected = True
return self.tests
else:
raise ValueError('Cannot collect tests of unknown type')
if not Options().args.reproduce:
color_stdout("Collecting tests in ", schema='ts_text')
color_stdout(
'%s (Found %s tests)' % (
repr(self.suite_path).ljust(16),
str(len(self.tests)).ljust(3)
),
schema='path'
)
color_stdout(": ", self.ini["description"], ".\n",
schema='ts_text')
self.tests_are_collected = True
return self.tests
class TestCommandLineParameters(unittest.TestCase):
# set test file location
cuboid_path = "cuboid.dat"
def setUp(self):
self.options = Options()
def test_defaults_options_are_set(self):
opts = self.options.parse()
self.assertEquals(opts.cuboid_file, self.cuboid_path)
def test_options_example_is_set(self):
opts = self.options.parse(['-f', 'foobar'])
self.assertEquals(opts.cuboid_file, 'foobar')
opts = self.options.parse(['--field', 'not-a-foobar'])
self.assertEquals(opts.cuboid_file, 'not-a-foobar')
def run_project(args):
options = Options()
options.parse(args[1:])
project = Survey(options)
print '\n--------------------------------Printing Survey Summary --------------------------------------\n'
print 'Printing date:', project.date()
# print 'Printing arg:', project.print_example_arg()
print '\n--------------------------------Printing Survey Question Details -----------------------------\n', project.parse_surveydata(
)
print '\n--------------------------------Printing Survey Responces Details ----------------------------\n', project.parse_responces(
)
print '\nSubmission Percentage: ', project.submission_percentage(), '%'
print '\nSubmission Count Total: ', project.submission_count_total()
print '\nAverage Rating Q1: ', project.avg_rating('ANS_Q1')
print '\nAverage Rating Q2: ', project.avg_rating('ANS_Q2')
print '\nAverage Rating Q3: ', project.avg_rating('ANS_Q3')
print '\nAverage Rating Q4: ', project.avg_rating('ANS_Q4')
print '\nAverage Rating Q5: ', project.avg_rating('ANS_Q5')
print '\n--------------------------------End Survey Summary -------------------------------------------\n'
def main(args):
options = Options()
config = options.parse(args[1:])
repo = GitRepository(os.path.curdir)
old_rev, new_rev, ref_name = sys.stdin.readline().split(' ')
if repo.is_revision_empty(old_rev):
old_rev = repo.get_default_branch_name()
if repo.is_revision_empty(new_rev):
# case of branch deletion
exit(0)
if config.php_cs_fixer_enabled:
fixer = PhpCsFixer(repo, config.php_cs_fixer_executable,
config.php_cs_fixer_config_path,
config.php_cs_fixer_dirs_to_create)
fixer.check_push(old_rev, new_rev)
each_commit_checker = EachCommitChecker(config, repo)
each_commit_checker.check(old_rev, new_rev)
def main_consistent():
color_stdout("Started {0}\n".format(" ".join(sys.argv)), schema='tr_text')
failed_test_ids = []
try:
main_loop_consistent(failed_test_ids)
except KeyboardInterrupt:
color_stdout('[Main loop] Caught keyboard interrupt\n',
schema='test_var')
except RuntimeError as e:
color_stdout("\nFatal error: %s. Execution aborted.\n" % e,
schema='error')
if Options().args.gdb:
time.sleep(100)
return -1
if failed_test_ids and Options().args.is_force:
color_stdout("\n===== %d tests failed:\n" % len(failed_test_ids),
schema='error')
for test_id in failed_test_ids:
color_stdout("----- %s\n" % str(test_id), schema='info')
return (-1 if failed_test_ids else 0)
def process_result(self, obj):
# Track tasks in progress.
if isinstance(obj, WorkerCurrentTask):
self.worker_current_task[obj.worker_id] = obj
# Skip irrelevant events.
if not isinstance(obj, WorkerOutput):
return
# Skip color_log() events if --debug is not passed.
if obj.log_only and not Options().args.debug:
return
self.warned_seconds_ago = 0.0
self.inactivity = 0.0
def __init__(self, ip, port):
Thread.__init__(self)
self.ip = ip
self.port = int(port)
if port == PROXY_OPTIONS.getValue('API_port'):
self.type = 'API'
# read client_ips, plainkeys from the key file, generate and store the AES-keys
self.AES_keys = Options.OptionReader(PROXY_OPTIONS.getValue('AES_key_file'))
for ip, plainkey in self.AES_keys.options.iteritems():
AES_key = SHA256.new()
AES_key.update(plainkey)
AES_key = AES_key.digest()
self.AES_keys.setValue(ip, AES_key)
if port == PROXY_OPTIONS.getValue('tracker_port'):
self.type = 'tracker'
self.start()
def main_loop_consistent(failed_test_ids):
# find and prepare all tasks/groups, print information
task_groups = get_task_groups().items()
print_greetings()
for name, task_group in task_groups:
# print information about current test suite
color_stdout("\n", '=' * 80, "\n", schema='separator')
color_stdout("TEST".ljust(48), schema='t_name')
color_stdout("PARAMS".ljust(16), schema='test_var')
color_stdout("RESULT\n", schema='test_pass')
color_stdout('-' * 75, "\n", schema='separator')
task_ids = task_group['task_ids']
show_reproduce_content = task_group['show_reproduce_content']
if not task_ids:
continue
worker_id = 1
worker = task_group['gen_worker'](worker_id)
for task_id in task_ids:
short_status = worker.run_task(task_id)
if short_status == 'fail':
reproduce_file_path = \
get_reproduce_file(worker.name)
color_stdout('Reproduce file %s\n' % reproduce_file_path,
schema='error')
if show_reproduce_content:
color_stdout("---\n", schema='separator')
print_tail_n(reproduce_file_path)
color_stdout("...\n", schema='separator')
failed_test_ids.append(task_id)
if not Options().args.is_force:
worker.stop_server(cleanup=False)
return
color_stdout('-' * 75, "\n", schema='separator')
worker.stop_server(silent=False)
color_stdout()
def __init__(self, mode, system_options=None):
self.mode = mode
# if we want to contact the Proxy, we need SystemOptions.txt for the AESkey and Proxy address:port
if self.mode == 'Proxy':
if system_options is not None:
self.SYSTEMoptions = system_options
required_keys = ('AES_key', 'proxy_address', 'API_port')
self.SYSTEMoptions.checkKeys(required_keys)
else:
raise Errors.FatalError(
msg=
'SystemOptions info not provided, only direct API calls possible'
)
# if we want to contact the API directly, we need ProxyOptions for the APIkeys and URLs
if self.mode == 'PPMS API':
try:
self.APIoptions = Options.OptionReader('ProxyOptions.txt')
except:
raise Errors.FatalError(
msg='ProxyOptions.txt is missing, only Proxy calls possible'
)
def check_tap_output(self):
""" Returns is_tap, is_ok """
try:
with open(self.tmp_result, 'r') as f:
content = f.read()
tap = pytap13.TAP13()
tap.parse(content)
except (ValueError, UnicodeDecodeError) as e:
color_stdout('\nTAP13 parse failed (%s).\n' % str(e),
schema='error')
color_stdout('\nNo result file (%s) found.\n' % self.result,
schema='error')
if not Options().args.update_result:
msg = 'Run the test with --update-result option to write the new result file.\n'
color_stdout(msg, schema='error')
self.is_crash_reported = True
return False, False
is_ok = True
for test_case in tap.tests:
if test_case.result == 'ok':
continue
if is_ok:
color_stdout('\n')
color_stdout(
'%s %s %s # %s %s\n' %
(test_case.result, test_case.id or '', test_case.description
or '-', test_case.directive or '', test_case.comment or ''),
schema='error')
if test_case.yaml:
self.tap_parse_print_yaml(test_case.yaml)
is_ok = False
if not is_ok:
color_stdout('Rejected result file: %s\n' % self.reject,
schema='test_var')
self.is_crash_reported = True
return True, is_ok
def setUp(self):
self.options = Options()
print(f'FB Bio: {active_users.number_of_users("bio")}, '
f'FB Che: {active_users.number_of_users("che")}, '
f'FB Phy: {active_users.number_of_users("phy")}')
print('--------------------')
print('Facility statistics:')
year = 2020
print(
f'The facility has given {active_users.number_of_trainings(year)} trainings in {year}.'
)
print(f'FB Bio: {active_users.number_of_trainings(year, "bio")}, '
f'FB Che: {active_users.number_of_trainings(year, "che")}, '
f'FB Phy: {active_users.number_of_trainings(year, "phy")}')
print('--------------------')
SYSTEM_OPTIONS = Options.OptionReader('SystemOptions.txt')
calling_mode = SYSTEM_OPTIONS.getValue('calling_mode')
with Timer('Getting facility system info from PPMS...',
'Microscope list populated!'):
equipment = Equipment()
with Timer('Getting active users from PPMS...', 'Active user list populated!'):
active_bic_users = ActiveBICUsers(equipment)
print_microscope_users(equipment.microscope_list)
print_user_info(active_bic_users.user_list)
print_facility_statistics(equipment, active_bic_users)
# for user_login in active_users:
# user = BICUser(user_login)
def main_loop_parallel():
color_stdout("Started {0}\n".format(" ".join(sys.argv)), schema='tr_text')
args = Options().args
jobs = args.jobs
if jobs < 1:
# faster result I got was with 2 * cpu_count
jobs = 2 * multiprocessing.cpu_count()
if jobs > 0:
color_stdout("Running in parallel with %d workers\n\n" % jobs,
schema='tr_text')
randomize = True
color_stdout("Timeout options:\n", schema='tr_text')
color_stdout('-' * 19, "\n", schema='separator')
color_stdout("REPLICATION_SYNC_TIMEOUT:".ljust(26) +
"{}\n".format(args.replication_sync_timeout),
schema='tr_text')
color_stdout("TEST_TIMEOUT:".ljust(26) + "{}\n".format(args.test_timeout),
schema='tr_text')
color_stdout("NO_OUTPUT_TIMEOUT:".ljust(26) +
"{}\n".format(args.no_output_timeout),
schema='tr_text')
color_stdout("\n", schema='tr_text')
task_groups = get_task_groups()
if Options().args.reproduce:
task_groups = reproduce_task_groups(task_groups)
jobs = 1
randomize = False
dispatcher = Dispatcher(task_groups, jobs, randomize)
dispatcher.start()
print_greetings()
color_stdout("\n", '=' * 86, "\n", schema='separator')
color_stdout("WORKR".ljust(6), schema='t_name')
color_stdout("TEST".ljust(48), schema='t_name')
color_stdout("PARAMS".ljust(16), schema='test_var')
color_stdout("RESULT\n", schema='test_pass')
color_stdout('-' * 81, "\n", schema='separator')
try:
is_force = Options().args.is_force
dispatcher.wait()
dispatcher.wait_processes()
color_stdout('-' * 81, "\n", schema='separator')
has_failed = dispatcher.statistics.print_statistics()
has_undone = dispatcher.report_undone(
verbose=bool(is_force or not has_failed))
if has_failed:
dispatcher.artifacts.save_artifacts()
return EXIT_FAILED_TEST
if has_undone:
return EXIT_NOTDONE_TEST
except KeyboardInterrupt:
color_stdout('-' * 81, "\n", schema='separator')
dispatcher.statistics.print_statistics()
dispatcher.report_undone(verbose=False)
raise
except HangError:
color_stdout('-' * 81, "\n", schema='separator')
dispatcher.statistics.print_statistics()
dispatcher.report_undone(verbose=False)
return EXIT_HANG
return EXIT_SUCCESS
# So we hack the open() builtin.
#
# https://stackoverflow.com/a/53347548/1598057
if PY3 and sys.version_info[0:2] < (3, 7):
std_open = __builtins__.open
def open_as_utf8(*args, **kwargs):
if len(args) >= 2:
mode = args[1]
else:
mode = kwargs.get('mode', '')
if 'b' not in mode:
kwargs.setdefault('encoding', 'utf-8')
return std_open(*args, **kwargs)
__builtins__.open = open_as_utf8
# don't sure why, but it values 1 or 2 gives 1.5x speedup for parallel
# test-run (and almost doesn't affect consistent test-run)
os.environ['OMP_NUM_THREADS'] = '2'
status = 0
force_parallel = bool(Options().args.reproduce)
if not force_parallel and Options().args.jobs == -1:
status = main_consistent()
else:
status = main_parallel()
exit(status)
data += packet
return data
encrypted_call = receive_data(self.connection)
iv2 = encrypted_call[:AES.block_size]
ciphered_msg = encrypted_call[AES.block_size:]
decryptor = AES.new(self.AES_key, AES.MODE_CFB, iv2)
decrypted_message = decryptor.decrypt(ciphered_msg)
data_from_client = pickle.loads(decrypted_message)
print self.address[0] + ' says ' + data_from_client[
'h'] + data_from_client['s']
server_response = {'h': 'Hello ', 'c': 'client'}
iv = Random.new().read(AES.block_size)
cipher = AES.new(self.AES_key, AES.MODE_CFB, iv)
pickled_response = pickle.dumps(server_response)
server_message = iv + cipher.encrypt(pickled_response)
data_to_client = struct.pack('>I',
len(server_message)) + server_message
self.connection.sendall(data_to_client)
self.connection.close()
proxy_options = Options.OptionReader('ServerOptions.txt')
PUMAPIproxy = ListeningSocket(proxy_options.getValue('host_ip'),
proxy_options.getValue('API_port'))
def run(args):
opts = Options()
opts.parse(args[1:])
w = Project(opts)
print w.welcome()
__author__ = 'nathan.muir'
from lib import Options, TaskMonitor
if __name__ == '__main__':
options = Options()
args = options.parse()
monitor = TaskMonitor(args)
monitor.run()
import sys
from lib import Scramble
from lib import Options
''' This utility prints all combination of characters of input text '''
if __name__ == '__main__':
options = Options()
opts = options.parse(sys.argv[1:])
v = Scramble(opts)
v.date()
v.print_example_arg()
v.scramble()
import sys
from lib import Options
if __name__ == '__main__':
options = Options()
if len(sys.argv[1:]) == 0:
options.help()
sys.exit(0)
opts = options.parse(sys.argv[1:])
if opts.input_file == None:
print "Please supply an input file"
sys.exit(1)
else:
print "At this point I'll process", opts.input_file
# TODO Create an object for the processer and pass
# the input file to it
sys.exit(0)
def _default_options(self): options = Options() opts, args = options.parse() return opts
