def combine_stations() -> Iterable[List[str]]:
'Combine the two railway station datasets by telecode.'
stations = AttrDict()
names = {}
stations_95306 = hyfw.dfs()
stations_12306 = kyfw.stations()
for s in stations_95306:
pinyin = s['PYM'].lower()
if len(pinyin) > 3:
pinyin = pinyin[:2] + pinyin[-1]
elif len(pinyin) < 3:
pinyin = ''
name, telecode = s['ZMHZ'], s['DBM']
stations[telecode] = [pinyin, name, s['TMIS'], s['SSJC']]
names[name] = telecode
for s in stations_12306:
old = stations.get(s.telecode)
new = [s.pinyin_code, s.name, '', '']
if s.name in names and s.telecode != names[s.name]:
conflict = '%s/%s -> %s/%s' % (names[s.name], new, s.telecode, old)
if s.telecode in stations:
conflict = 'Name conflict: %s -> ?' % conflict
else:
conflict = 'Solved conflict: %s' % conflict
old = stations[s.telecode] = stations.pop(names[s.name])
old[:2] = new[:2]
elif s.telecode in stations and s.name != stations[s.telecode][1]:
new[-2] = tmis(s.name).get(s.name, '')
conflict = '%s/(%s => %s)' % (s.telecode, old, new)
if new[-2] and old[-2] != new[-2]: # TMIS codes conflict
conflict = 'Ambiguous telecode: %s' % conflict
else:
conflict = 'Solved conflict: %s' % conflict
old[:2] = new[:2]
else:
if s.telecode not in stations:
stations[s.telecode] = [''] * 4
stations[s.telecode][:2] = new[:2]
continue
# resolve merge conflicts manually
shell(dict(vars(), s=stations), '\n%s' % conflict)
for k, v in stations.items():
# drop telecodes with spaces
# so those can be used as temporary names in conflict solving
v.insert(2, '' if ' ' in k else k)
yield v
def main():
'The entrypoint.'
x = API()
if x.is_logged_in():
print('Already logged in.')
else:
x.show_captcha()
coordinates = x.input_captcha()
x.check_captcha(coordinates)
x.login(username=input('Login: '******'Password: '******'SJP', 'WCN')[-1]
print(dict(x.left_tickets(Z53[0])))
shell(vars())
def install_collector(client, collector, params):
fail = False
installer = download_installer(client, collector, params)
# ensure installer is executable
os.chmod(installer, 0755)
logging.debug('installing ' + str(installer))
result = util.shell([str(installer), ' -y'])
if result['code'] != 0 or result['stderr'] != '':
err = result['stderr']
# if we failed but there's no stderr, set err msg to stdout
if err == '':
err = result['stdout']
logging.debug('Collector install failed')
logging.debug('stdout: ' + str(result['stdout']))
logging.debug('stderr: ' + str(result['stderr']))
logging.debug('Cleaning up collector install directory')
util.remove_path(config.INSTALL_PATH + config.AGENT_DIRECTORY)
fail = True
# be nice and clean up
logging.debug('Cleaning up downloaded installer')
util.remove_path(installer)
if fail:
util.fail(err)
def install_collector(client, collector, params):
fail = False
installer = download_installer(client, collector, params)
# ensure installer is executable
os.chmod(installer, 0755)
logging.debug('installing ' + str(installer))
result = util.shell([str(installer), ' -y'])
if result['code'] != 0 or result['stderr'] != '':
err = result['stderr']
# if we failed but there's no stderr, set err msg to stdout
if err == '':
err = result['stdout']
logging.debug('Collector install failed')
logging.debug('stdout: ' + str(result['stdout']))
logging.debug('stderr: ' + str(result['stderr']))
logging.debug('Cleaning up collector install directory')
util.remove_path(config.INSTALL_PATH + config.AGENT_DIRECTORY)
fail = True
# be nice and clean up
logging.debug('Cleaning up downloaded installer')
util.remove_path(installer)
if fail:
util.fail(err)
def test_util_shell(self):
if self._test_exec("test_shell"):
Prg = self.Prg
if self.Prg[
"OsIsUnixBased"]: # the tests with ls/grep don't work on Windows
Result = util.shell(
f"ls {Prg['DirPrgRoot']} | grep READ").strip()
self.assertEqual(Result, "README.md")
def heuristic_search(stations, initials=None) -> Iterable[List[str]]:
'Search the TMIS database using name initials.'
# create indexes for faster lookup
names, tmis_codes = ({s[field]: index
for index, s in enumerate(stations)}
for field in (1, -2))
if not initials:
initials = {name[0] for name in names}.union({s[-1] for s in stations})
for initial in initials:
progress()
for name, tmis_code in tmis.dfs(initial).items():
# append as a new station
if name not in names and tmis_code not in tmis_codes:
yield ['', name, '', tmis_code, '']
# replace in-place
elif name in names:
old = stations[names[name]]
if not old[-2]:
old[-2] = tmis_code
elif old[-2] != tmis_code:
conflict = 'TMIS code conflict: %s' % old
shell(dict(vars(), s=stations), '\n%s' % conflict)
def install_collector(client, collector, params):
fail = False
installer = download_installer(client, collector, params)
# ensure installer is executable
os.chmod(installer, 0755)
install_cmd = [str(installer), '-y']
# force update the collector object to ensure all details are up to date
# e.g. build version
collector = find_collector_by_id(client, collector.id)
# if this is a newer installer that defaults to non-root user, force root
logging.debug('Collector version ' + str(collector.build))
if int(collector.build) >= MIN_NONROOT_INSTALL_VER or params['use_ea']:
install_cmd.extend(['-u', 'root'])
proxy = util.parse_proxy(params)
if proxy is not None:
proxy_addr = proxy['host_addr']
proxy_user = proxy['user']
proxy_pass = proxy['pass']
install_cmd.extend(['-p', proxy_addr])
if proxy_user is not None and proxy_user != '':
install_cmd.extend(['-U', proxy_user])
if proxy_pass is not None and proxy_pass != '':
install_cmd.extend(['-P', proxy_pass])
result = util.shell(install_cmd)
if result['code'] != 0 or result['stderr'] != '':
err = result['stderr']
# if we failed but there's no stderr, set err msg to stdout
if err == '':
err = result['stdout']
logging.debug('Collector install failed')
logging.debug('stdout: ' + str(result['stdout']))
logging.debug('stderr: ' + str(result['stderr']))
logging.debug('Cleaning up collector install directory')
# util.remove_path(config.INSTALL_PATH + config.AGENT_DIRECTORY)
fail = True
# be nice and clean up
logging.debug('Cleaning up downloaded installer')
# util.remove_path(installer)
if fail:
util.fail(err)
def load_hbase(colo, action, start, end, ids, offset):
cmds = CMD.split(' ')
cmds.append(colo)
cmds.append(action)
cmds.append(str(start))
cmds.append(str(end))
cmds.append(ids)
ret = util.shell(cmds)
ret = ret.split(b'\n')[-1]
ret = util.str_to_json(ret)
if (action == 'cache'):
for item in ret:
values = item['value']
for value in values:
value['cacheId'] = str(int(value['cacheId']) + offset)
return ret
for index, s in enumerate(stations)}
for field in (1, -2))
if not initials:
initials = {name[0] for name in names}.union({s[-1] for s in stations})
for initial in initials:
progress()
for name, tmis_code in tmis.dfs(initial).items():
# append as a new station
if name not in names and tmis_code not in tmis_codes:
yield ['', name, '', tmis_code, '']
# replace in-place
elif name in names:
old = stations[names[name]]
if not old[-2]:
old[-2] = tmis_code
elif old[-2] != tmis_code:
conflict = 'TMIS code conflict: %s' % old
shell(dict(vars(), s=stations), '\n%s' % conflict)
if __name__ == '__main__':
stations = list(combine_stations())
stations.extend(heuristic_search(stations))
shell(dict(vars(), s=stations), 'Well done.')
with open(path, 'w') as f:
print(dump_stations(stations), file=f)
print('Dumped %d stations to "%s".' % (len(stations), path))
def install_collector(client, collector, params):
fail = False
current_version = collector.build
installer, err = download_installer(client, collector, params)
if not installer and err == 'version error':
# get the latest stable release if not force require EA version;
collector.build = params['collector_version'] = 0
log_msg = 'retry to get latest available collector version'
logging.debug(log_msg)
installer, err = download_installer(client, collector, params)
if not installer:
util.fail(err)
# ensure installer is executable
os.chmod(installer, 0755)
install_cmd = [str(installer), '-y']
# force update the collector object to ensure all details are up to date
# e.g. build version
if collector.build != 0:
collector = find_collector_by_id(client, collector.id)
logging.debug('Collector version ' + str(collector.build))
# if this is a newer installer that defaults to non-root user, force root
if int(collector.build) >= MIN_NONROOT_INSTALL_VER or params[
'use_ea'] or int(collector.build == 0):
install_cmd.extend(['-u', 'root'])
proxy = util.parse_proxy(params)
if proxy is not None:
proxy_addr = proxy['host_addr']
proxy_user = proxy['user']
proxy_pass = proxy['pass']
install_cmd.extend(['-p', proxy_addr])
if proxy_user is not None and proxy_user != '':
install_cmd.extend(['-U', proxy_user])
if proxy_pass is not None and proxy_pass != '':
install_cmd.extend(['-P', proxy_pass])
result = util.shell(install_cmd)
if result['code'] != 0 or result['stderr'] != '':
err = result['stderr']
# if we failed but there's no stderr, set err msg to stdout
if err == '':
err = result['stdout']
# check for false fail condition
success_msg = 'LogicMonitor Collector has been installed successfully'
if err.lower().strip().strip('\n') == 'unknown option: u' and \
success_msg.lower() in str(result['stdout']).lower():
fail = False
else:
logging.debug('Collector install failed')
logging.debug('stdout: ' + str(result['stdout']))
logging.debug('stderr: ' + str(result['stderr']))
logging.debug('Cleaning up collector install directory')
# util.remove_path(config.INSTALL_PATH + config.AGENT_DIRECTORY)
fail = True
# be nice and clean up
logging.debug('Cleaning up downloaded installer')
# util.remove_path(installer)
if fail:
util.fail(err)
else:
# log message if version is outdated
try:
if os.path.isfile(config.INSTALL_STAT_PATH):
with open(config.INSTALL_STAT_PATH) as install_stat:
for line in install_stat:
if 'complexInfo=' in line:
key, val = line.partition("=")[::2]
complexInfo = json.loads(val.strip('\n'))
collector_version = complexInfo['collector'][
'version'].strip()
if str(collector_version) != str(current_version):
upgrade_msg = 'Requested collector version %s ' \
'is outdated so upgraded to %s' \
' version '
upgrade_msg = upgrade_msg % (str(
current_version), str(collector_version))
logging.info(upgrade_msg)
except:
pass
for t_count in num_thread_range:
binExec = '{}/{}/{}/{}'.format(BINDIR, BENCHMARKS[b_name]['type'],
b_name, binname)
flags = list(" -t " + str(t_count) + \
" -i " + str(iterations) + \
" -e " + str(executions))
if args.runs:
flags.append(" -r ")
if args.overview:
flags.append(" -o ")
if args.useful_work_in:
flags.append(" -u " + str(uins))
if args.useful_work_out:
flags.append(" -y " + str(uouts))
ret = shell(lkwd_cmd + binExec + "".join(flags))
log_file = '{}/{}.out'.format(bin_dir, binname)
if BENCHMARKS[b_name]['useful']:
with open(log_file, 'w') as f:
print(ret, file=f)
else:
current_path = os.path.dirname(os.path.realpath(__file__))
currentFiles = os.listdir(current_path)
for f in currentFiles:
if f.endswith("producers.txt") or f.endswith(
"consumers.txt") or f.endswith(
"pop.txt") or f.endswith("push.txt"):
shutil.move(f, bin_dir)
elif BENCHMARKS[b_name]['runType'] is 'CMD_Args':
"""
Benchmark configurations
"""
import sys
from util import shell
from util import fail
if sys.version_info[0] < 3:
fail('Python 3 is required to run this code')
CORES = shell('lscpu | gawk \'/^CPU\(s\):/{print $2}\'')
THREADS_PER_CORE = shell('lscpu | gawk \'/^Thread\(s\) per core:/{print $4}\'')
PHYSICAL_CORES = int(CORES) // int(THREADS_PER_CORE)
BENCHMARKS = {
'ArrayLock': {
'type': 'blocking',
'bins': [
'arraylock_sc',
'arraylock_ar',
],
'useful': True,
'runType': 'cxxopts',
},
'CLHQueueLock': {
'type': 'blocking',
'bins': [
'clhqueue_lock_sc',
'clhqueue_lock_ar',
],