def postproduction(self, records):
IMAGES_DIR = self.IMAGES_DIR
for r in records:
for k, v in r.items():
if 'date' in k or 'time' in k:
if r[k]:
r[k] = to_date(v)
img_fn = r['image']
img_src = os.path.join(BASE, IMAGES_DIR, img_fn)
assert os.path.exists(img_src), img_src
img_dst_folder = os.path.join(MEDIA_DIR, IMAGES_DIR)
img_dst = os.path.join(img_dst_folder, img_fn)
if not os.path.exists(img_dst_folder):
os.makedirs(os.path.join(img_dst_folder))
if not os.path.exists(img_dst):
cmd = 'cp "%s" "%s"' % (img_src, img_dst)
print cmd
sh(cmd)
r['image'] = os.path.join(IMAGES_DIR, img_fn)
print r['image']
from import_journalentry import OUT_FN as fn_journalentries_json
journalentries = json.load(open(fn_journalentries_json))
for je in journalentries:
fr = je['fields']['folio_number_from']
fr = int(fr)
to = je['fields']['folio_number_to']
to = int(to)
if fr <= int(r['folio_number']) <= to:
r['journalentry'] = je['pk']
return records
def read_hk_items():
fn = os.path.join(BASE, HARTAKARUN, FN_HKITEM)
# images_dir = os.path.join(BASE, HARTAKARUN, 'HK Category Images')
lines = csv.reader(codecs.open(fn, encoding='iso-8859-1'), delimiter=';')
lines = list(lines)
records = []
headers = lines[0]
print headers
headers = [MAP_FIELDS_TO_MODEL_HKITEM[k] for k in headers]
for l in lines[1:]:
l = [unicode(x) for x in l]
r = dict(zip(headers, l))
if r['image'] == 'dummy.png':
r['image'] = None
else:
sub_dir = r['image'][:3]
sh('cp "%s" "%s"' %
(os.path.join(IMAGES_DIR, sub_dir, r['image']), MEDIA_DIR))
records.append(r)
for x in records:
print x
print headers
return records
def postproduction(self, records):
IMAGES_DIRS = self.IMAGES_DIRS
print IMAGES_DIRS
fn = os.path.join(BASE, 'retrobooks/RB items', 'retrobookDuplicateScanReferenceInstance_20120524.csv')
lines = csv.reader(codecs.open(fn, encoding=ENCODING), delimiter=DELIMITER)
scanid2retrobookid = dict([(l[2], l[1]) for l in lines])
for i in range(0, len(records)):
r = records[i]
r['retrobook'] = scanid2retrobookid[r['pk']]
if int(r['retrobook']) >= 5:
r['retrobook'] = None
if r['retrobook']:
img_fn = r['image']
img_src = os.path.join(BASE,'retrobooks/RB items', IMAGES_DIRS[r['retrobook']], 'CD FolioImages', img_fn)
img_dst_folder = os.path.join(MEDIA_DIR, IMAGES_DIRS[r['retrobook']], 'CD FolioImages')
if not os.path.exists(img_src):
print 'could not find', img_src
img_dst = os.path.join(img_dst_folder, img_fn)
print img_dst
if not os.path.exists(img_dst) :
cmd = 'cp "%s" "%s"' % (img_src, img_dst)
print cmd
sh(cmd)
r['image'] = os.path.join(IMAGES_DIRS[r['retrobook']], 'CD FolioImages', img_fn)
return records
def processNextEvent():
maxEventId = int(
json.loads(
sh(f'kubectl -n {env.CD_NAMESPACE} get configmap -ojson ' +
getFullName('event-cursor')))['data']['eventID'])
events = sh(
f"kubectl -n {env.CD_NAMESPACE} get configmaps -o=jsonpath='{{.items[*].metadata.name}}'"
+
f" -lowner=quickcd,kind=GitHubEvent,status=pending,org={env.CD_GITHUB_ORG_NAME},repo={env.CD_GITHUB_REPO_NAME}"
).strip()
if not events:
return False
eventIds = [
eid for eid in (int(e.split('-').pop()) for e in events.split(' '))
if eid <= maxEventId
]
# run nonblocking handlers first if we can
for eid in eventIds:
eventResource = json.loads(
sh(f'kubectl -n {env.CD_NAMESPACE} get configmap -o=json ' +
getFullName(eid)))
if runHandlers(json.loads(eventResource['data']['event']), False, eid,
eventResource['metadata']['labels']):
return True
earliestEventId = min(eventIds)
eventResource = json.loads(
sh(f'kubectl -n {env.CD_NAMESPACE} get configmap -o=json ' +
getFullName(earliestEventId)))
return runHandlers(json.loads(eventResource['data']['event']), True,
earliestEventId, eventResource['metadata']['labels'])
def generateKubeconfig():
sh(f'bx login --apikey {env.CD_BX_TOKEN} -a https://api.ng.bluemix.net')
sh(f'bx cs region-set {env.CD_REGION_DASHED}')
confPath = sh(f'bx cs cluster-config --export {env.CD_CLUSTER_NAME}'
).split('=').pop()
sh(f'ln -s {os.path.dirname(confPath)} $HOME/.kube'
) # some certs in that dir along with config
sh(f'mv $HOME/.kube/{os.path.basename(confPath)} $HOME/.kube/config')
def run(num_servers = None, # default 5
client_command = 'build/Examples/SmokeTest',
timeout = 10):
if num_servers is None:
num_servers = min(5, len(smokehosts))
server_ids = range(1, num_servers + 1)
with Sandbox() as sandbox:
sh('rm -rf log')
sh('rm -f debug/*')
sh('scripts/initlog.py --serverid 1 --address %s' % smokehosts[0][0])
for server_id in server_ids:
host = smokehosts[server_id - 1]
command = 'build/LogCabin --id %d --config smoketest.conf' % server_id
print('Starting %s on %s' % (command, smokehosts[server_id - 1][0]))
sandbox.rsh(smokehosts[server_id - 1][0], command, bg=True,
stderr=open('debug/%d' % server_id, 'w'))
sandbox.checkFailures()
print('Growing cluster')
sh('build/Examples/Reconfigure %s' %
' '.join([h[0] for h in smokehosts]))
print('Starting %s on localhost' % client_command)
client = sandbox.rsh('localhost', client_command, bg=True,
stderr=open('debug/client', 'w'))
start = time.time()
while client.proc.returncode is None:
sandbox.checkFailures()
time.sleep(.1)
if time.time() - start > timeout:
raise Exception('timeout exceeded')
def read_items(self, limit=None):
print '/home/jelle/Dropbox/DASA Beta/Testdata/Bookshelves/ANRI_HR859/ANRI_HR859 FolioImages'
images = []
for dirname in self.SOURCE_DIRS:
images += [os.path.join(dirname, fn) for fn in os.listdir(dirname)]
records = []
images.sort()
backcovers = []
frontcovers = []
for img in images:
if img.endswith('backcover.png'):
backcovers.append(img)
elif img.endswith('cover.png'):
frontcovers.append(img)
images = frontcovers + [
x for x in images if x not in backcovers and x not in frontcovers
] + backcovers
for i, img in enumerate(images):
# if 'cover.png' in img:
# continue
relative_path_to_local_copy_of_image = os.path.join(
img.split('/')[-3],
img.split('/')[-1])
dir = os.path.split(
os.path.join(MEDIA_DIR,
relative_path_to_local_copy_of_image))[0]
if not os.path.exists(dir):
os.makedirs(dir)
sh('cp "%s" "%s"' % (img, dir))
pagenumber = img.split('.')[0].split('f')[-1]
try:
pagenumber = int(pagenumber)
pagenumber = str(pagenumber)
except:
pagenumber = ''
record = {
'pk': str(i + 1),
'institution': 'ANRI',
'fonds': 'HR',
'file_id': img.split('/')[-3].split('_')[1][2:],
'image': relative_path_to_local_copy_of_image,
'position': str(i + 1),
'pagenumber': pagenumber,
}
record['retrobook'] = file_ids2pk[record['file_id']]
records.append(record)
return records
def read_category_instance():
fn = os.path.join(BASE, HARTAKARUN, FN_CATEGORYINSTANCE)
images_dir = os.path.join(BASE, HARTAKARUN, 'HK Category Images')
print 'reading', fn
# lines = csv.reader(codecs.open(fn, encoding='utf8'), delimiter=';')
lines = unicode_csv_reader(fn)
lines = [x for x in lines]
headers = lines[0]
headers = [MAP_FIELDS_TO_MODEL[k] for k in headers]
records_category = []
records_maincategory = []
assert os.path.exists(images_dir), images_dir
for l in lines[1:]:
# l = [to_unicode(x) for x in l]
r = dict(zip(headers, l))
if r['name_en'] in ['None']:
continue
if r['hartakarun_main_category'] == '0':
r['hartakarun_main_category'] = None
if not r['image'] or r['image'] == 'dummy.png':
r['image'] = 'HK Category I.6.png'
sh('cp "%s" "%s"' % (os.path.join(images_dir, r['image']), MEDIA_DIR))
if r['image_intro'] == 'dummy.png':
r['image_intro'] = None
if not r['image_intro']:
r['image_intro'] = r['image']
else:
cmd = 'cp "%s" "%s"' % (os.path.join(images_dir,
r['image_intro']), MEDIA_DIR)
sh(cmd)
print cmd
if r['hartakarun_main_category']:
records_category.append(r)
else:
del r['hartakarun_main_category']
records_maincategory.append(r)
# assert type(r['position']) == type(0), r['position']
# for x in records:
# print x
print headers
return records_maincategory, records_category
def read_hk_scans():
fn = os.path.join(BASE, HARTAKARUN, FN_HKSCAN)
# images_dir = os.path.join(BASE, HARTAKARUN, 'HK Category Images')
lines = csv.reader(codecs.open(fn, encoding='iso-8859-1'), delimiter=';')
lines = list(lines)
records = []
headers = lines[0]
print headers
headers = [MAP_FIELDS_TO_MODEL_HKSCAN[k] for k in headers]
for l in lines[1:]:
l = [unicode(x) for x in l]
r = dict(zip(headers, l))
records.append(r)
if int(r['hartakarun_item']) <= 3:
sub_dir = 'HK%s/HK%s Folio Images' % (r['hartakarun_item'],
r['hartakarun_item'])
image_fns = [
'ID_ANRI_HR_%s_%s.png' % (r['file_id'], r['position']),
'ID_ANRI_HR_%s_%s.jpg' % (r['file_id'], r['position']),
]
for image_fn in image_fns:
src = os.path.join(IMAGES_DIR, sub_dir, image_fn)
if os.path.exists(src):
break
# src = os.path.join(IMAGES_DIR, sub_dir, image_fn)
dst = os.path.join(MEDIA_DIR, sub_dir, image_fn)
try:
os.makedirs(os.path.join(MEDIA_DIR, sub_dir))
except OSError:
#probably because it already existed
pass
cmd = 'cp "%s" "%s"' % (src, dst)
print cmd
sh(cmd)
r['image'] = dst
if int(r['hartakarun_item']) > 3:
r['hartakarun_item'] = None
for x in records:
print x
print headers
return records
def runHandlers(event, blocking, eventID=0, labels={}):
# now that we have the event of interest, fire all the (remaining) event handlers for it.
workPerformed = False
for handler in dispatchTable[event['type']]:
if handler.isBlocking == blocking and handler.filterFn(
event['payload']) and handler.id not in labels:
if eventID:
lastRun = float(labels.get(f'{handler.id}_last_run', '0'))
attempts = int(labels.get(f'{handler.id}_attempts', '0'))
if (time.time() - lastRun) / 60 < attempts**3:
continue
writeLabels(
eventID, **{
f'{handler.id}_last_run': time.time(),
f'{handler.id}_attempts': attempts + 1
})
workPerformed = True
# reset workspace and call handler
sh('rm -rf /tmp')
sh('mkdir -m 777 /tmp')
os.chdir('/tmp')
setCurrentHandlerFnName(handler.name)
print(f"Event {eventID}. Calling handler: {handler.name}")
try:
handler.handlerFn(event['payload'])
except:
print(traceback.format_exc())
if blocking:
return True # don't run other handlers for this event since encountered an error
else:
continue # run remaining nonblocking handlers for this evt
if eventID:
writeLabels(eventID, **{f'{handler.id}': 'complete'})
# check if any handlers remaining and mark complete if not
if eventID:
allDone = True
for handler in dispatchTable[event['type']]:
if handler.filterFn(event['payload']) and handler.id not in labels:
allDone = False
break
if allDone:
writeLabels(eventID, status='handled')
return workPerformed
def read_items():
fn = os.path.join(BASE, FN)
# images_dir = os.path.join(BASE, HARTAKARUN, 'HK Category Images')
lines = csv.reader(codecs.open(fn, encoding=ENCODING), delimiter=DELIMITER)
lines = list(lines)
records = []
headers = lines[0]
print headers
headers = [MAP_FIELDS_TO_MODEL[k] for k in headers]
resolutions = read_resolutions()
for l in lines[1:]:
l = [unicode(x) for x in l]
r = dict(zip(headers, l))
for k, v in r.items():
if 'date' in k or 'time' in k:
r[k] = to_date(v)
img_fn = r['image']
img_src = os.path.join(BASE, IMAGES_DIR, img_fn)
assert os.path.exists(img_src), img_src
img_dst_folder = os.path.join(MEDIA_DIR, IMAGES_DIR)
img_dst = os.path.join(img_dst_folder, img_fn)
if not os.path.exists(img_dst_folder):
os.makedirs(os.path.join(img_dst_folder))
if not os.path.exists(img_dst):
cmd = 'cp "%s" "%s"' % (img_src, img_dst)
print cmd
sh(cmd)
r['image'] = os.path.join(IMAGES_DIR, img_fn)
try:
resolution = resolutions[r['file_id']]
r['resolution'] = resolution['pk']
except KeyError:
r['resolution'] = None
records.append(r)
for x in records:
print x
print headers
return records
def main():
arguments = docopt(__doc__)
client_command = arguments['--client']
num_servers = int(arguments['--servers'])
reconf_opts = arguments['--reconf']
if reconf_opts == "''":
reconf_opts = ""
timeout = int(arguments['--timeout'])
server_ids = range(1, num_servers + 1)
with Sandbox() as sandbox:
sh('rm -rf smoketeststorage/')
sh('rm -f debug/*')
sh('mkdir -p debug')
sh('scripts/initlog.py '
'--serverid 1 '
'--address %s '
'--storage smoketeststorage' % smokehosts[0][0])
for server_id in server_ids:
host = smokehosts[server_id - 1]
command = 'build/LogCabin --id %d --config smoketest.conf' % server_id
print('Starting %s on %s' % (command, smokehosts[server_id - 1][0]))
sandbox.rsh(smokehosts[server_id - 1][0], command, bg=True,
stderr=open('debug/%d' % server_id, 'w'))
sandbox.checkFailures()
print('Growing cluster')
sh('build/Examples/Reconfigure %s %s' %
(reconf_opts, ' '.join([h[0] for h in smokehosts[:num_servers]])))
print('Starting %s on localhost' % client_command)
client = sandbox.rsh('localhost', client_command, bg=True,
stderr=open('debug/client', 'w'))
start = time.time()
while client.proc.returncode is None:
sandbox.checkFailures()
time.sleep(.1)
if time.time() - start > timeout:
raise Exception('timeout exceeded')
def main():
arguments = docopt(__doc__)
client_command = arguments['--client']
server_command = arguments['--binary']
num_servers = int(arguments['--servers'])
reconf_opts = arguments['--reconf']
if reconf_opts == "''":
reconf_opts = ""
timeout = int(arguments['--timeout'])
server_ids = range(1, num_servers + 1)
with Sandbox() as sandbox:
sh('rm -rf smoketeststorage/')
sh('rm -f debug/*')
sh('mkdir -p debug')
print('Initializing first server\'s log')
sandbox.rsh(smokehosts[0][0],
'%s --bootstrap --id 1 --config smoketest.conf' %
server_command,
stderr=open('debug/bootstrap', 'w'))
print()
for server_id in server_ids:
host = smokehosts[server_id - 1]
command = ('%s --id %d --config smoketest.conf' %
(server_command, server_id))
print('Starting %s on %s' %
(command, smokehosts[server_id - 1][0]))
sandbox.rsh(smokehosts[server_id - 1][0],
command,
bg=True,
stderr=open('debug/%d' % server_id, 'w'))
sandbox.checkFailures()
print('Growing cluster')
sh('build/Examples/Reconfigure %s %s' %
(reconf_opts, ' '.join([h[0] for h in smokehosts[:num_servers]])))
print('Starting %s on localhost' % client_command)
client = sandbox.rsh('localhost',
client_command,
bg=True,
stderr=open('debug/client', 'w'))
start = time.time()
while client.proc.returncode is None:
sandbox.checkFailures()
time.sleep(.1)
if time.time() - start > timeout:
raise Exception('timeout exceeded')
def setupGit():
sh(f'git config --global user.email "{env.CD_EMAIL_ADDRESS}"')
sh('git config --global user.name "quickcd"')
# allow git to work without ssh for private repos
if 'CD_GITHUB_TOKEN' in env:
sh("git config --global url.'https://%s:x-oauth-basic@%s/'.insteadOf 'https://%s/'"
% (env.CD_GITHUB_TOKEN, env.CD_GITHUB_DOMAIN, env.CD_GITHUB_DOMAIN))
def main():
arguments = docopt(__doc__)
client_command = arguments["--client"]
server_command = arguments["--binary"]
num_servers = int(arguments["--servers"])
reconf_opts = arguments["--reconf"]
if reconf_opts == "''":
reconf_opts = ""
timeout = int(arguments["--timeout"])
server_ids = range(1, num_servers + 1)
with Sandbox() as sandbox:
sh("rm -rf smoketeststorage/")
sh("rm -f debug/*")
sh("mkdir -p debug")
print("Initializing first server's log")
sandbox.rsh(
smokehosts[0][0],
"%s --bootstrap --id 1 --config smoketest.conf" % server_command,
stderr=open("debug/bootstrap", "w"),
)
print()
for server_id in server_ids:
host = smokehosts[server_id - 1]
command = "%s --id %d --config smoketest.conf" % (server_command, server_id)
print("Starting %s on %s" % (command, smokehosts[server_id - 1][0]))
sandbox.rsh(smokehosts[server_id - 1][0], command, bg=True, stderr=open("debug/%d" % server_id, "w"))
sandbox.checkFailures()
print("Growing cluster")
sh("build/Examples/Reconfigure %s %s" % (reconf_opts, " ".join([h[0] for h in smokehosts[:num_servers]])))
print("Starting %s on localhost" % client_command)
client = sandbox.rsh("localhost", client_command, bg=True, stderr=open("debug/client", "w"))
start = time.time()
while client.proc.returncode is None:
sandbox.checkFailures()
time.sleep(0.1)
if time.time() - start > timeout:
raise Exception("timeout exceeded")
def start(self, directory):
perf_dir = '%s/perf' % directory
self.perf_dir = perf_dir
common.pdsh(self.nodes, 'mkdir -p -m0755 -- %s' % perf_dir).communicate()
perf_template = 'perf {} &'.format(self.args_template)
local_node = common.get_localnode(self.nodes)
if local_node:
for pid_path in glob.glob(os.path.join(self.pid_dir, 'osd.*.pid')):
with open(pid_path) as pidfile:
pid = pidfile.read().strip()
perf_cmd = perf_template.format(perf_dir=perf_dir, pid=pid)
runner = common.sh(local_node, perf_cmd)
self.perf_runners.append(runner)
else:
# ${pid} will be handled by remote's sh
perf_cmd = perf_template.format(perf_dir=perf_dir, pid='${pid}')
common.pdsh(self.nodes, ['for pid in `cat %s/osd.*.pid`;' % self.pid_dir,
'do', perf_cmd,
'done'])
import sys
sys.path.append('./import_scripts')
from common import sh
ls = sh('git log | grep Date')
ls = ls.split('\n')
for x in ls:
print x
print 'SINCE FIRST COMMIT:'
print '-' * 20
print len(ls), 'commits'
print len(ls) * 2, 'hours, given 2 hours per commit'
ls = [x[0:18] for x in ls]
ls = set(ls)
ls = list(set(ls))
print len(ls), 'days'
print len(ls) * 6, 'hours in 6 hour workdays'
print '-' * 20
print 'SINCE 1 OCTOBER:'
ls = sh('git log --since={2012-10-01} | grep Date')
ls = ls.split('\n')
print '-' * 20
print len(ls), 'commits'
print len(ls) * 2, 'hours, given 2 hours per commit'
ls = [x[0:18] for x in ls]
def main():
arguments = docopt(__doc__)
client_command = arguments['--client']
server_command = arguments['--binary']
num_servers = int(arguments['--servers'])
reconf_opts = arguments['--reconf']
if reconf_opts == "''":
reconf_opts = ""
timeout = int(arguments['--timeout'])
server_ids = range(1, num_servers + 1)
cluster = "--cluster=%s" % ','.join([h[0] for h in
smokehosts[:num_servers]])
alphabet = [chr(ord('a') + i) for i in range(26)]
cluster_uuid = ''.join([random.choice(alphabet) for i in range(8)])
with Sandbox() as sandbox:
sh('rm -rf smoketeststorage/')
sh('rm -f debug/*')
sh('mkdir -p debug')
for server_id in server_ids:
host = smokehosts[server_id - 1]
with open('smoketest-%d.conf' % server_id, 'w') as f:
f.write('serverId = %d\n' % server_id)
f.write('listenAddresses = %s\n' % host[0])
f.write('clusterUUID = %s\n' % cluster_uuid)
f.write('snapshotMinLogSize = 1024')
f.write('\n\n')
try:
f.write(open('smoketest.conf').read())
except:
pass
print('Initializing first server\'s log')
sandbox.rsh(smokehosts[0][0],
'%s --bootstrap --config smoketest-%d.conf' %
(server_command, server_ids[0]),
stderr=open('debug/bootstrap', 'w'))
print()
for server_id in server_ids:
host = smokehosts[server_id - 1]
command = ('%s --config smoketest-%d.conf' %
(server_command, server_id))
print('Starting %s on %s' % (command, host[0]))
sandbox.rsh(host[0], command, bg=True,
stderr=open('debug/%d' % server_id, 'w'))
sandbox.checkFailures()
print('Growing cluster')
sh('build/Examples/Reconfigure %s %s set %s' %
(cluster,
reconf_opts,
' '.join([h[0] for h in smokehosts[:num_servers]])))
print('Starting %s %s on localhost' % (client_command, cluster))
client = sandbox.rsh('localhost',
'%s %s' % (client_command, cluster),
bg=True,
stderr=open('debug/client', 'w'))
start = time.time()
while client.proc.returncode is None:
sandbox.checkFailures()
time.sleep(.1)
if time.time() - start > timeout:
raise Exception('timeout exceeded')
#!/usr/bin/env python
from common import sh
sh("""
set -x
set -e
rm -rf /etc/apache2/sites-enabled/
rm -rf /etc/apache2/sites-available/
rm -rf /etc/apache2/sites-includes/
ln -s /yerk-data/etc/apache2/sites-enabled/ /etc/apache2/sites-enabled/
ln -s /yerk-data/etc/apache2/sites-available/ /etc/apache2/sites-available/
ln -s /yerk-data/etc/apache2/site-includes/ /etc/apache2/site-includes/
a2enmod headers
a2enmod ssl
a2enmod dav
a2enmod dav_fs
a2enmod proxy
a2enmod proxy_connect
/etc/init.d/apache2 restart
""")
m = re.search('Running for election in term (\d+)', line)
if m is not None:
b['wake'] = int(m.group(1))
terms = [b['term'] for b in server_beliefs.values()]
leaders = [b['leader'] for b in server_beliefs.values()]
if same(terms) and terms[0] > after_term:
assert same(leaders), server_beliefs
return {'leader': leaders[0],
'term': terms[0],
'num_woken': sum([1 for b in server_beliefs.values() if b['wake'] > after_term])}
else:
time.sleep(.25)
sandbox.checkFailures()
with Sandbox() as sandbox:
sh('rm -f debug/*')
sh('mkdir -p debug')
server_ids = range(1, num_servers + 1)
servers = {}
def start(server_id):
host = hosts[server_id - 1]
command = 'build/LogCabin -i %d' % server_id
print('Starting LogCabin -i %d on %s' % (server_id, host[0]))
server = sandbox.rsh(host[0], command, bg=True,
stderr=open('debug/%d' % server_id, 'w'))
servers[server_id] = server
for server_id in server_ids:
start(server_id)
def main():
arguments = docopt(__doc__)
client_commands = arguments['--client']
server_command = arguments['--binary']
num_servers = int(arguments['--servers'])
reconf_opts = arguments['--reconf']
if reconf_opts == "''":
reconf_opts = ""
timeout = int(arguments['--timeout'])
killinterval = int(arguments['--killinterval'])
launchdelay = int(arguments['--launchdelay'])
server_ids = range(1, num_servers + 1)
cluster = "--cluster=%s" % ','.join(
[h[0] for h in smokehosts[:num_servers]])
with Sandbox() as sandbox:
sh('rm -rf smoketeststorage/')
sh('rm -f debug/*')
sh('mkdir -p debug')
for server_id in server_ids:
host = smokehosts[server_id - 1]
with open('smoketest-%d.conf' % server_id, 'w') as f:
try:
f.write(open('smoketest.conf').read())
f.write('\n\n')
except:
pass
f.write('serverId = %d\n' % server_id)
f.write('listenAddresses = %s\n' % host[0])
print('Initializing first server\'s log')
sandbox.rsh(smokehosts[0][0],
'%s --bootstrap --config smoketest-%d.conf' %
(server_command, server_ids[0]),
stderr=open('debug/bootstrap', 'w'))
print()
processes = {}
def launch_server(server_id):
host = smokehosts[server_id - 1]
command = ('%s --config smoketest-%d.conf -l %s' %
(server_command, server_id, 'debug/%d' % server_id))
print('Starting %s on %s' % (command, host[0]))
processes[server_id] = sandbox.rsh(host[0], command, bg=True)
sandbox.checkFailures()
for server_id in server_ids:
launch_server(server_id)
print('Growing cluster')
sh('build/Examples/Reconfigure %s %s set %s' %
(cluster, reconf_opts, ' '.join(
[h[0] for h in smokehosts[:num_servers]])))
for i, client_command in enumerate(client_commands):
print('Starting %s %s on localhost' % (client_command, cluster))
sandbox.rsh('localhost',
'%s %s' % (client_command, cluster),
bg=True,
stderr=open('debug/client%d' % i, 'w'))
start = time.time()
lastkill = start
tolaunch = [] # [(time to launch, server id)]
while True:
time.sleep(.1)
sandbox.checkFailures()
now = time.time()
if now - start > timeout:
print('Timeout met with no errors')
break
if now - lastkill > killinterval:
server_id = random.choice(processes.keys())
print('Killing server %d' % server_id)
sandbox.kill(processes[server_id])
del processes[server_id]
lastkill = now
tolaunch.append((now + launchdelay, server_id))
while tolaunch and now > tolaunch[0][0]:
launch_server(tolaunch.pop(0)[1])
def main():
arguments = docopt(__doc__)
growing=False
if arguments['--growing']:
growing=True
sharedfs=False
if arguments['--sharedfs']:
sharedfs=True
client_command = 'build/Examples/Benchmark'
server_command = 'build/LogCabin'
num_servers = len(smokehosts)
server_ids = range(1, num_servers + 1)
cluster = "--cluster=%s" % ','.join([h[1] for h in
smokehosts[:num_servers]])
with Sandbox() as sandbox:
sh('rm -rf debug/')
sh('mkdir -p debug')
if growing:
for server_id in server_ids:
host = smokehosts[server_id - 1]
with open('benchtest-%d.conf' % server_id, 'w') as f:
f.write('serverId = %d\n' % server_id)
f.write('listenAddresses = %s\n' % host[1])
f.write('storagePath = %s'
%os.path.join(os.getcwd(),'teststorage/'))
f.write('\n\n')
try:
f.write(open('benchtest.conf').read())
except:
pass
if not sharedfs:
print('Copying script files to remote servers',
'and removing previous storage')
for server_id in server_ids:
host = smokehosts[server_id - 1]
sh('scp benchtest-%d.conf %s:%s'
%(server_id, host[0],os.getcwd()))
sandbox.rsh(host[0], 'rm -rf teststorage')
print()
else:
sh('rm -rf teststorage')
print('Initializing first server\'s log')
sandbox.rsh(smokehosts[0][0],
'%s --bootstrap --config benchtest-%d.conf' %
(server_command, server_ids[0]),
stderr=open('debug/bootstrap_err', 'w'))
print()
for server_id in server_ids:
host = smokehosts[server_id - 1]
command = ('%s --config benchtest-%d.conf' %
(server_command, server_id))
print('Starting %s on %s' % (command, host[1]))
sandbox.rsh(host[0], command, bg=True,
stderr=open('debug/%derr' % server_id, 'w'))
sandbox.checkFailures()
print('Growing cluster')
#Carreful at verbosity here
sh('build/Examples/Reconfigure --verbosity=ERROR %s set %s' %
(cluster,
' '.join([h[1] for h in smokehosts[:num_servers]])))
lat_tp_test(client_command, cluster, num_servers)
def lat_tp_test(client_command, cluster, num_servers):
rps_res = "/home/raho/logcabin/results/rps_res%d"%num_servers
lat_res = "/home/raho/logcabin/results/lat_res%d"%num_servers
waits = [0, 2500, 5000, 10000, 25000, 50000, 75000, 100000]
#[0, 50, 100, 250, 500, 1000, 5000, 10000, 20000, 30000]
opts = "--timeout=0 --writes=1600000 --size=1024 --threads=250"
lat_opts = "--writes=100000 --size=1024 --wait=500000 --threads=1"
binary = os.path.basename(client_command)
sh('echo start >%s'%rps_res)
sh('echo start >%s'%lat_res)
for w in waits:
# to be determined better
timeout = 15
print('wait with %dus'%w)
#Run all threads for throughput
sh('echo wait with %dus >>%s'%(w,rps_res))
sh(('%s %s --wait=%d %s 1>>%s 2>/dev/null &'
% (client_command, opts, w, cluster,
rps_res
)))
time.sleep(1)
sh('echo wait with %dus >>%s'%(w,lat_res))
sh(('%s %s --timeout=%d %s 1>>%s 2>/dev/null'
% (client_command, lat_opts, timeout, cluster,
lat_res
)))
sh(('killall %s'% binary))
if w is not waits[-1]:
print('sleeping..')
time.sleep(6)
sh('echo end >>%s'%rps_res)
sh('echo end >>%s'%lat_res)
def fetchAndSaveNewEvents():
# todo: respect the x-poll-interval header
# first run don't save anything, second run, even if no id saved previously, save all
try:
resource = json.loads(
sh(f"kubectl -n {env.CD_NAMESPACE} get ConfigMap {getFullName('event-cursor')} -ojson"
))
except:
firstRun = True
fetchedETag = '"none"'
fetchedEventID = 0
print("Can't retrieve config, assumiing first run.")
else:
firstRun = False
fetchedETag = resource['data']['ETag']
fetchedEventID = int(resource['data']['eventID'])
newETag = fetchedETag
newEventID = fetchedEventID
url = env.CD_REPO_API_URL + '/events'
events = []
for i in range(15): # Current GH limit is 10 pages
if i > 10:
raise Exception(
"Github only provides 10 pages of results, something's wrong with pagination."
)
resp = http.request(
'GET',
url,
headers=dict(http.headers, **{'If-None-Match': fetchedETag})
if i == 0 and not firstRun else http.headers)
if resp.status == 304:
print("Fetching events 304 (nothing new)")
return
else:
checkResponse(resp)
print(f"Processing page {i+1}")
if i == 0 and 'ETag' in resp.headers:
newETag = resp.headers['ETag']
tmpEvents = json.loads(resp.data.decode('utf-8'))
tmpEventsFiltered = [
e for e in tmpEvents if int(e['id']) > fetchedEventID
]
events += tmpEventsFiltered
if len(tmpEvents) != len(tmpEventsFiltered):
break # already reached event we saved last
if firstRun:
break #during first run only want latest id
linkKey = '; rel="next"'
if 'Link' in resp.headers and linkKey in resp.headers['Link']:
url = resp.headers['Link'].split(linkKey)[0].split(
',').pop().strip('< >')
else:
break
if events:
newEventID = int(
events[0]['id']
) # may be not an event that we saved, we save only filtered ones we're listening for
else:
print('Interesting, no events at all!') # new repo or old repo?
# only save events on subsequent runs
if not firstRun:
# get rid of duplicates and filter
eventDict = dict((e['id'], e) for e in events if any(
handler.filterFn(e['payload'])
for handler in dispatchTable[e['type']]))
if eventDict:
# use apply in case this command worked but saving cursor failed, resulting in resave
# todo: not sure why yapf is using 3 space indenting here and below
sh(f'kubectl -n {env.CD_NAMESPACE} apply -f-',
input=json.dumps(dict(
kubeList,
items=[
dict(kubeConfigMap,
metadata={
'name': getFullName(e['id']),
'labels': {
'owner': 'quickcd',
'kind': 'GitHubEvent',
'org': env.CD_GITHUB_ORG_NAME,
'repo': env.CD_GITHUB_REPO_NAME,
'status': 'pending'
}
},
data={
'event':
json.dumps(e,
ensure_ascii=False,
allow_nan=False)
}) for e in eventDict.values()
]),
ensure_ascii=False,
allow_nan=False))
# below we use create for first run to make sure we don't accidentally override a config that existed but failed to load above
sh(f"kubectl -n {env.CD_NAMESPACE} {'create --save-config' if firstRun else 'apply'} -f-",
input=json.dumps(dict(kubeConfigMap,
metadata={
'name': getFullName('event-cursor'),
'labels': {
'owner': 'quickcd'
}
},
data={
'eventID': str(newEventID),
'ETag': newETag
}),
ensure_ascii=False,
allow_nan=False))
def main():
arguments = docopt(__doc__)
client_command = arguments['--client']
server_command = arguments['--binary']
num_servers = int(arguments['--servers'])
reconf_opts = arguments['--reconf']
if reconf_opts == "''":
reconf_opts = ""
timeout = int(arguments['--timeout'])
server_ids = range(1, num_servers + 1)
cluster = "--cluster=%s" % ','.join(
[h[0] for h in smokehosts[:num_servers]])
alphabet = [chr(ord('a') + i) for i in range(26)]
cluster_uuid = ''.join([random.choice(alphabet) for i in range(8)])
with Sandbox() as sandbox:
sh('rm -rf smoketeststorage/')
sh('rm -f debug/*')
sh('mkdir -p debug')
for server_id in server_ids:
host = smokehosts[server_id - 1]
with open('smoketest-%d.conf' % server_id, 'w') as f:
f.write('serverId = %d\n' % server_id)
f.write('listenAddresses = %s\n' % host[0])
f.write('clusterUUID = %s\n' % cluster_uuid)
f.write('snapshotMinLogSize = 1024')
f.write('\n\n')
try:
f.write(open('smoketest.conf').read())
except:
pass
print('Initializing first server\'s log')
sandbox.rsh(smokehosts[0][0],
'%s --bootstrap --config smoketest-%d.conf' %
(server_command, server_ids[0]),
stderr=open('debug/bootstrap', 'w'))
print()
for server_id in server_ids:
host = smokehosts[server_id - 1]
command = ('%s --config smoketest-%d.conf' %
(server_command, server_id))
print('Starting %s on %s' % (command, host[0]))
sandbox.rsh(host[0],
command,
bg=True,
stderr=open('debug/%d' % server_id, 'w'))
sandbox.checkFailures()
print('Growing cluster')
sh('build/Examples/Reconfigure %s %s set %s' %
(cluster, reconf_opts, ' '.join(
[h[0] for h in smokehosts[:num_servers]])))
print('Starting %s %s on localhost' % (client_command, cluster))
client = sandbox.rsh('localhost',
'%s %s' % (client_command, cluster),
bg=True,
stderr=open('debug/client', 'w'))
start = time.time()
while client.proc.returncode is None:
sandbox.checkFailures()
time.sleep(.1)
if time.time() - start > timeout:
raise Exception('timeout exceeded')
leaders[0],
'term':
terms[0],
'num_woken':
sum([
1 for b in server_beliefs.values()
if b['wake'] > after_term
])
}
else:
time.sleep(.25)
sandbox.checkFailures()
with Sandbox() as sandbox:
sh('rm -f debug/*')
sh('mkdir -p debug')
server_ids = range(1, num_servers + 1)
servers = {}
def start(server_id):
host = hosts[server_id - 1]
command = 'build/LogCabin -i %d' % server_id
print('Starting LogCabin -i %d on %s' % (server_id, host[0]))
server = sandbox.rsh(host[0],
command,
bg=True,
stderr=open('debug/%d' % server_id, 'w'))
servers[server_id] = server
def main():
arguments = docopt(__doc__)
client_command = arguments['--client']
server_command = arguments['--binary']
num_servers = int(arguments['--servers'])
reconf_opts = arguments['--reconf']
if reconf_opts == "''":
reconf_opts = ""
client_opts = arguments['--clientops']
if client_opts == "''":
client_opts = ""
it_num = int(arguments['--it'])
max_ns = len(smokehosts)
if max_ns < num_servers:
print('Number of servers in config file: %s\n'%max_ns)
num_servers = max_ns
del max_ns
sharedfs=False
if arguments['--sharedfs']:
sharedfs=True
server_ids = range(1, num_servers + 1)
cluster = "--cluster=%s" % ','.join([h[1] for h in
smokehosts[:num_servers]])
with Sandbox() as sandbox:
sh('rm -rf debug/')
sh('mkdir -p debug')
sh('echo %s > debug/bench_cluster'%cluster)
for server_id in server_ids:
host = smokehosts[server_id - 1]
with open('smoketest-%d.conf' % server_id, 'w') as f:
f.write('serverId = %d\n' % server_id)
f.write('listenAddresses = %s\n' % host[1])
f.write('storagePath = %s'
%os.path.join(os.getcwd(),'teststorage/'))
f.write('\n\n')
try:
f.write(open('smoketest.conf').read())
except:
pass
if not sharedfs:
print('Copying script files to remote servers',
'and removing previous storage')
for server_id in server_ids:
host = smokehosts[server_id - 1]
sh('scp smoketest-%d.conf %s:%s'
%(server_id, host[0],os.getcwd()))
sandbox.rsh(host[0], 'rm -rf teststorage')
print()
else:
sh('rm -rf teststorage')
print('Initializing first server\'s log')
sandbox.rsh(smokehosts[0][0],
'%s --bootstrap --config smoketest-%d.conf' %
(server_command, server_ids[0]),
stderr=open('debug/bootstrap_err', 'w'))
print()
for server_id in server_ids:
host = smokehosts[server_id - 1]
command = ('%s --config smoketest-%d.conf' %
(server_command, server_id))
print('Starting %s on %s' % (command, host[1]))
sandbox.rsh(host[0], command, bg=True,
stderr=open('debug/%derr' % server_id, 'w'))
sandbox.checkFailures()
print('Growing cluster')
#Carreful at verbosity here
sh('build/Examples/Reconfigure --verbosity=ERROR %s %s set %s' %
(cluster,
reconf_opts,
' '.join([h[1] for h in smokehosts[:num_servers]])))
print('Starting %s %s on localhost over %d iterations'
% (client_command, cluster, it_num))
print('\n')
# time.sleep(100)
for i in range(0,it_num):
# client = sandbox.rsh('localhost',
# ('%s %s %s'
# % (client_command, client_opts, cluster)),
# bg=False, stderr=open('debug/client', 'w')
# )
# small hack to redirect results to stderr for clean files
sh(('%s %s %s 1>&2 2>/dev/null'
% (client_command, client_opts, cluster)))
m = re.search('Running for election in term (\d+)', line)
if m is not None:
b['wake'] = int(m.group(1))
terms = [b['term'] for b in server_beliefs.values()]
leaders = [b['leader'] for b in server_beliefs.values()]
if same(terms) and terms[0] > after_term:
assert same(leaders), server_beliefs
return {'leader': leaders[0],
'term': terms[0],
'num_woken': sum([1 for b in server_beliefs.values() if b['wake'] > after_term])}
else:
time.sleep(.25)
sandbox.checkFailures()
with Sandbox() as sandbox:
sh('rm -f debug/*')
server_ids = range(1, num_servers + 1)
servers = {}
def start(server_id):
host = hosts[server_id - 1]
command = 'build/LogCabin -i %d' % server_id
print('Starting LogCabin -i %d on %s' % (server_id, host[0]))
server = sandbox.rsh(host[0], command, bg=True,
stderr=open('debug/%d' % server_id, 'w'))
servers[server_id] = server
for server_id in server_ids:
start(server_id)
def main():
arguments = docopt(__doc__)
client_command = arguments['--client']
server_command = arguments['--binary']
num_servers = int(arguments['--servers'])
reconf_opts = arguments['--reconf']
if reconf_opts == "''":
reconf_opts = ""
timeout = int(arguments['--timeout'])
killinterval = int(arguments['--killinterval'])
launchdelay = int(arguments['--launchdelay'])
server_ids = range(1, num_servers + 1)
cluster = "--cluster=%s" % ','.join([h[0] for h in
smokehosts[:num_servers]])
with Sandbox() as sandbox:
sh('rm -rf smoketeststorage/')
sh('rm -f debug/*')
sh('mkdir -p debug')
for server_id in server_ids:
host = smokehosts[server_id - 1]
with open('smoketest-%d.conf' % server_id, 'w') as f:
try:
f.write(open('smoketest.conf').read())
f.write('\n\n')
except:
pass
f.write('serverId = %d\n' % server_id)
f.write('listenAddresses = %s\n' % host[0])
print('Initializing first server\'s log')
sandbox.rsh(smokehosts[0][0],
'%s --bootstrap --config smoketest-%d.conf' %
(server_command, server_ids[0]),
stderr=open('debug/bootstrap', 'w'))
print()
processes = {}
def launch_server(server_id):
host = smokehosts[server_id - 1]
command = ('%s --config smoketest-%d.conf -l %s' %
(server_command, server_id, 'debug/%d' % server_id))
print('Starting %s on %s' % (command, host[0]))
processes[server_id] = sandbox.rsh(
host[0], command, bg=True)
sandbox.checkFailures()
for server_id in server_ids:
launch_server(server_id)
print('Growing cluster')
sh('build/Examples/Reconfigure %s %s set %s' %
(cluster,
reconf_opts,
' '.join([h[0] for h in smokehosts[:num_servers]])))
print('Starting %s %s on localhost' % (client_command, cluster))
client = sandbox.rsh('localhost',
'%s %s' % (client_command, cluster),
bg=True,
stderr=open('debug/client', 'w'))
start = time.time()
lastkill = start
tolaunch = [] # [(time to launch, server id)]
while True:
time.sleep(.1)
sandbox.checkFailures()
now = time.time()
if now - start > timeout:
print('Timeout met with no errors')
break
if now - lastkill > killinterval:
server_id = random.choice(processes.keys())
print('Killing server %d' % server_id)
sandbox.kill(processes[server_id])
del processes[server_id]
lastkill = now
tolaunch.append((now + launchdelay, server_id))
while tolaunch and now > tolaunch[0][0]:
launch_server(tolaunch.pop(0)[1])
