def random_annotation(self):
p = gen_id(self.namespace)
i = gen_id(self.namespace)
b = [[123,234], [345,456]] # tuples do not survive JSON roundtripping
t = gen_id(self.namespace)
a = gen_id(self.namespace)
ts = iso8601()
return structs(timestamp=ts, pid=p, image=i, geometry=b, category=t, annotator=a)
def create_work():
print 'creating work'
for n in range(5):
ping_pid = gen_id()
job_pid = gen_id()
print 'pinging: %s <- %s' % (job_pid, ping_pid)
assert isok(client.depend(job_pid, ping_pid, PING_ROLE))
print 'waking up'
assert isok(client.wakeup(WAKEUP_KEY))
def timestamped_output_of(cmd,stdout=True,jobid=None):
issued = iso8601ms()
if jobid is None:
jobid = gen_id()
def log_struct(message):
return {
'command': cmd,
'issued': issued,
'jobid': jobid,
'message': message,
'timestamp': iso8601ms()
}
yield log_struct('executing shell command %s, jobid %s' % (cmd,jobid))
if stdout:
p = Popen(cmd, shell=True, stdout=PIPE)
else:
p = Popen(cmd, shell=True, stderr=PIPE)
while True:
if stdout:
line = p.stdout.readline()
else:
line = p.stderr.readline()
if not line:
p.wait()
if p.returncode != 0:
message = 'process exited abnormally with exit code %d: %s' % (p.returncode, cmd)
yield log_struct(message)
raise RuntimeError(message)
yield log_struct('process closed output stream: '+cmd)
return
yield log_struct(line.rstrip('\n'))
def run_callback(self,message):
jobid = gen_id()[:5]
def selflog(line):
self.log('%s %s' % (jobid, line))
def self_check_log(line,bin_pid):
selflog(line)
self.output_check -= 1
if self.output_check <= 0:
if self.exists(bin_pid):
selflog('STOPPING JOB - %s completed by another worker' % bin_pid)
raise JobExit(bin_pid, SKIP)
self.output_check = CHECK_EVERY
bin_pid = message
dest_file = self.storage.dest(bin_pid)
if self.exists(bin_pid):
selflog('SKIPPING %s - already completed' % bin_pid)
return SKIP
job_dir = os.path.join(self.config.tmp_dir, gen_id())
try:
os.makedirs(job_dir)
except:
selflog('WARNING cannot create temporary directory %s' % job_dir)
tmp_file = os.path.join(job_dir, self.storage.zipname(bin_pid))
matlab = Matlab(self.config.matlab_exec_path,self.config.matlab_path,output_callback=lambda l: self_check_log(l, bin_pid))
cmd = 'bin_blobs(\'%s\',\'%s\')' % (bin_pid, job_dir)
try:
self.output_check = CHECK_EVERY
matlab.run(cmd)
if not os.path.exists(tmp_file):
selflog('WARNING bin_blobs succeeded but produced no output for %s' % bin_pid)
elif not self.exists(bin_pid): # check to make sure another worker hasn't finished it in the meantime
if self.deposit is not None:
selflog('DEPOSITING blob zip for %s to deposit service' % bin_pid)
self.deposit.deposit(bin_pid,tmp_file)
else:
selflog('SAVING completed blob zip for %s to %s' % (bin_pid, dest_file))
local_deposit(bin_pid,tmp_file)
else:
selflog('NOT SAVING - blobs for %s already present at output destination' % bin_pid)
except KeyboardInterrupt:
selflog('KeyboardInterrupt, requeueing job before exit')
return DIE
finally:
try:
shutil.rmtree(job_dir)
except:
selflog('WARNING cannot remove temporary directory %s' % job_dir)
def annotations_for(file):
for raw in csv.DictReader(fin,['bin','roi','category','annotator']):
yield {
PID: gen_id(ANNOTATION_NAMESPACE),
IMAGE: DATA_NAMESPACE + raw['bin'].replace('.mat','_') + raw['roi'],
TIMESTAMP: iso8601(),
CATEGORY: raw['category'],
ANNOTATOR: raw['annotator'],
}
def add_user(self,username, password, **kvs):
salt = gen_id() # generate salt
kvs['annotator_id'] = username
kvs['passwd'] = md5_string(salt + password) # md5 salt + password is the encrypted credential
kvs['salt'] = salt
kvs = kvs.items()
ks = ','.join([k for k,_ in kvs])
ss = ','.join(['%s' for _ in kvs])
vs = [v for _,v in kvs]
with xa(self.config.psql_connect) as (connection,cursor):
cursor.execute('insert into auth (' + ks + ') values (' + ss + ')',vs)
connection.commit()
def preflight(self):
for p in self.config.matlab_path:
assert os.path.exists(p)
assert os.path.exists(self.config.tmp_dir)
try:
tempfile = os.path.join(self.config.tmp_dir,gen_id()+'.txt')
with open(tempfile,'w') as tf:
tf.write('test')
tf.flush()
assert os.path.exists(tempfile)
os.remove(tempfile)
assert not os.path.exists(tempfile)
except:
raise
def test_list_annotations(self):
with app.test_request_context():
ns = [0,1,2,3,100]
image_pids = [gen_id(self.namespace) for _ in range(len(ns))]
for (n,image_pid) in zip(ns,image_pids):
# create n annotations for this image pid
for _ in range(n):
ann = self.random_annotation()
ann.image = image_pid
self.app.post(url_for('create_annotations', pid=ann.pid), data=jsons([ann]))
for (n,image_pid) in zip(ns,image_pids):
ann_list = structs(self.app.get(url_for('list_annotations', image_pid=image_pid)).data)
# FIXME don't just check the length of the result, check the contents
assert len(ann_list) == n
def preflight(self):
for p in self.config.matlab_path:
if not os.path.exists(p):
raise
if not os.path.exists(self.config.tmp_dir):
raise
tempfile = os.path.join(self.config.tmp_dir,gen_id()+'.txt')
with open(tempfile,'w') as tf:
tf.write('test')
tf.flush()
if not os.path.exists(tempfile):
raise
os.remove(tempfile)
if os.path.exists(tempfile):
raise
def win_callback(self,params):
start_time = params['start_time']
end_time = iso8601()
img_in = params['img_in']
tmp_out = params['img_out'] # temporary output file
img_out = params['final_out'] # final output file
process_id = gen_id()
if not os.path.exists(tmp_out):
self.log('FAIL temporary output file does not exist: %s' % tmp_out)
return FAIL
in_md5 = md5_file(img_in)
out_md5 = md5_file(tmp_out)
in_length = os.stat(img_in).st_size
out_length = os.stat(tmp_out).st_size
used = {
'process_id': process_id,
'algorithm_id': '201203_ic', # FIXME uncontrolled
'direction': 'used', # FIXME local id
'pathname': img_in, # FIXME local pathname
'no_earlier_than': start_time,
'no_later_than': end_time,
'fixity_md5': in_md5,
'fixity_length': in_length
}
generated_by = {
'process_id': process_id,
'algorithm_id': '201203_ic', # FIXME uncontrolled
'direction': 'generated by', # FIXME local id
'pathname': img_out, # FIXME local pathname
'no_earlier_than': start_time,
'no_later_than': end_time,
'fixity_md5': out_md5,
'fixity_length': out_length
}
# FIXME emit provenance record
prov_qname = '%s_prov' % self.qname
try:
self.enqueue(json.dumps(used), prov_qname)
self.enqueue(json.dumps(generated_by), prov_qname)
except:
raise JobExit('Failed to enqueue provenance records', FAIL)
try:
os.rename(tmp_out, img_out)
except:
raise JobExit('Cannot move temporary file into place: %s -> %s' % (tmp_out, img_out), FAIL)
return WIN
def deposit_impl(product,pid):
destpath = get_destpath(product, pid)
product_data = request.data
destpath_part = '%s_%s.part' % (destpath, gen_id())
try:
os.makedirs(os.path.dirname(destpath))
except:
pass
with open(destpath_part,'w') as out:
shutil.copyfileobj(StringIO(product_data), out)
os.rename(destpath_part, destpath)
utcnow = iso8601utcnow()
message = '%s wrote %d bytes to %s' % (utcnow, len(product_data), destpath)
return jsonr(dict(
status='OK',
time=utcnow,
message=message,
pid=pid,
path=destpath
))
def deposit(pid):
req = DashboardRequest(pid, request)
try:
destpath = files.get_product_destination(session, pid)
except NotFound:
abort(404)
product_data = request.data
destpath_part = '%s_%s.part' % (destpath, gen_id())
try:
os.makedirs(os.path.dirname(destpath))
except:
pass
with open(destpath_part,'w') as out:
shutil.copyfileobj(StringIO(product_data), out)
os.rename(destpath_part, destpath)
utcnow = iso8601()
message = '%s wrote %d bytes to %s' % (utcnow, len(product_data), destpath)
return Response(json.dumps(dict(
status='OK',
time=utcnow,
message=message,
pid=pid,
path=destpath
)), mimetype=MIME_JSON)
def extract_blobs(self,bin_pid):
try:
jobid = self.config.task_id
except:
jobid = gen_id()[:5]
def selflog(line):
self.log('[%s] %s' % (jobid, line))
def self_check_log(line,bin_pid):
selflog(line)
now = time.time()
elapsed = now - self.last_check
self.last_check = now
if elapsed > CHECK_EVERY:
if self.exists(bin_pid):
msg = 'STOPPING JOB - %s completed by another worker' % bin_pid
selflog(msg)
raise JobExit(msg, SKIP)
if self.exists(bin_pid):
selflog('SKIPPING %s - already completed' % bin_pid)
return SKIP
job_dir = os.path.join(self.config.tmp_dir, gen_id())
zip_dir = os.path.join(self.config.tmp_dir, gen_id())
bin_zip_path = os.path.join(zip_dir, binzipname(bin_pid))
try:
os.makedirs(job_dir)
selflog('CREATED temporary directory %s for %s' % (job_dir, bin_pid))
except:
selflog('WARNING cannot create temporary directory %s for %s' % (job_dir, bin_pid))
try:
os.makedirs(zip_dir)
selflog('CREATED temporary directory %s for %s' % (zip_dir, bin_pid))
except:
selflog('WARNING cannot create temporary directory %s for %s' % (zip_dir, bin_pid))
try:
selflog('LOADING and STITCHING %s' % bin_pid)
with open(bin_zip_path,'wb') as binzip:
targets = represent.binpid2zip(bin_pid, binzip, resolver=self.resolver)
if len(targets)==0:
selflog('SKIPPING %s - no targets in bin' % bin_pid)
return SKIP
tmp_file = os.path.join(job_dir, zipname(bin_pid))
matlab = Matlab(self.config.matlab_exec_path,self.config.matlab_path,output_callback=lambda l: self_check_log(l, bin_pid))
cmd = 'bin_blobs(\'%s\',\'%s\',\'%s\')' % (bin_pid, bin_zip_path, job_dir)
self.output_check = CHECK_EVERY
matlab.run(cmd)
if not os.path.exists(tmp_file):
selflog('WARNING bin_blobs succeeded but no output file found at %s' % tmp_file)
elif not self.exists(bin_pid): # check to make sure another worker hasn't finished it in the meantime
selflog('DEPOSITING blob zip for %s to deposit service at %s' % (bin_pid, self.config.blob_deposit))
self.deposit.deposit(bin_pid,tmp_file)
selflog('DEPOSITED blob zip for %s ' % bin_pid)
else:
selflog('NOT SAVING - blobs for %s already present at output destination' % bin_pid)
except KeyboardInterrupt:
selflog('KeyboardInterrupt, exiting')
return DIE
except JobExit:
pass
finally:
try:
shutil.rmtree(job_dir)
selflog('DELETED temporary directory %s for %s' % (job_dir, bin_pid))
except:
selflog('WARNING cannot remove temporary directory %s for %s' % (job_dir, bin_pid))
try:
shutil.rmtree(zip_dir)
selflog('DELETED temporary directory %s for %s' % (zip_dir, bin_pid))
except:
selflog('WARNING cannot remove temporary directory %s for %s' % (zip_dir, bin_pid))
selflog('DONE - no more actions for %s' % bin_pid)
from oii.ifcb2.orm import User, Role
from oii.ifcb2.dashboard.flasksetup import user_manager
if __name__=='__main__':
print 'Searching for default admin user in database ...'
user = session.query(User).filter_by(email='*****@*****.**').first()
if user is not None:
print 'Found user. Deleting ...'
session.delete(user)
session.commit()
else:
print 'Not found.'
print 'Creating recovery user/password ...'
username = '******'
password = gen_id()[:8]
u = User(
first_name='Admin',
last_name='User',
email=username,
username=username,
password=user_manager.hash_password(password),
is_enabled=True
)
r = session.query(Role).filter_by(name='Admin').first()
u.roles.append(r)
session.add(u)
session.commit()
print 'User %s created with password %s' % (username, password)
def __init__(self,qname,host='localhost'):
self.host = host
self.qname = qname
self.log_channel = None
self.work_channels = {}
self.workerid = ('%s_%s' % (gen_id()[:4], platform.node()))
import sys
from oii.utils import gen_id
from oii.ifcb2.session import dbengine, session
from oii.ifcb2.orm import User, Role
from oii.ifcb2.dashboard.flasksetup import user_manager
if __name__ == '__main__':
try:
password = sys.argv[1]
except IndexError:
password = gen_id()[:8]
try:
email = sys.argv[2]
except IndexError:
email = '*****@*****.**'
user = session.query(User).filter_by(email=email).first()
if user is not None:
session.delete(user)
session.commit()
u = User(first_name='Admin',
last_name='User',
email=email,
username=email,
password=user_manager.hash_password(password),
is_enabled=True)
r = session.query(Role).filter_by(name='Admin').first()
u.roles.append(r)
session.add(u)
def extract_features(self,bin_pid):
jobid = gen_id()[:5]
def selflog(line):
self.log('[%s] %s' % (jobid, line))
def self_check_log(line,bin_pid):
selflog(line)
now = time.time()
elapsed = now - self.last_check
self.last_check = now
if elapsed > CHECK_EVERY:
if self.complete(bin_pid):
msg = 'STOPPING JOB - %s completed by another worker' % bin_pid
selflog(msg)
raise JobExit(msg, SKIP)
if self.complete(bin_pid):
selflog('SKIPPING %s - already completed' % bin_pid)
return SKIP
job_dir = os.path.join(self.config.tmp_dir, gen_id())
zip_dir = os.path.join(self.config.tmp_dir, gen_id())
bin_zip_path = os.path.join(zip_dir, binzipname(bin_pid))
try:
os.makedirs(job_dir)
selflog('CREATED temporary directory %s for %s' % (job_dir, bin_pid))
except:
selflog('WARNING cannot create temporary directory %s for %s' % (job_dir, bin_pid))
try:
os.makedirs(zip_dir)
selflog('CREATED temporary directory %s for %s' % (zip_dir, bin_pid))
except:
selflog('WARNING cannot create temporary directory %s for %s' % (zip_dir, bin_pid))
selflog('LOADING and STITCHING %s' % bin_pid)
with open(bin_zip_path,'wb') as binzip:
represent.binpid2zip(bin_pid, binzip, resolver=self.resolver)
blobzipurl = bin_pid + '_blob.zip'
blobzipfile = re.sub(r'\.zip','_blob.zip',bin_zip_path)
selflog('LOADING blob zip from %s -> %s' % (blobzipurl, blobzipfile))
drain(UrlSource(blobzipurl), LocalFileSink(blobzipfile))
feature_csv = os.path.join(job_dir, csvname(bin_pid))
multiblob_csv = os.path.join(job_dir, 'multiblob', multiblobname(bin_pid))
matlab = Matlab(self.config.matlab_exec_path,self.config.matlab_path,output_callback=lambda l: self_check_log(l, bin_pid))
namespace = os.path.dirname(bin_zip_path) + '/'
lid = os.path.basename(bin_zip_path)
cmd = 'bin_features(\'%s\',\'%s\',\'%s\',\'chatty\')' % (namespace, lid, job_dir + '/')
selflog('RUNNING %s' % cmd)
try:
self.output_check = CHECK_EVERY
matlab.run(cmd)
if not os.path.exists(feature_csv):
msg = 'WARNING bin_features succeeded but no output file found at %s' % feature_csv
selflog(msg)
raise JobExit(msg,FAIL)
if not self.complete(bin_pid): # check to make sure another worker hasn't finished it in the meantime
selflog('DEPOSITING features csv for %s to deposit service at %s' % (bin_pid, self.config.features_deposit))
self.deposit.deposit(bin_pid,feature_csv)
selflog('DEPOSITED features csv for %s ' % bin_pid)
if os.path.exists(multiblob_csv):
selflog('DEPOSITING multiblob csv for %s to deposit service at %s' % (bin_pid, self.config.features_deposit))
self.multiblob_deposit.deposit(bin_pid,multiblob_csv)
selflog('DEPOSITED multiblob csv for %s ' % bin_pid)
else:
selflog('NOT SAVING - features for %s already present at output destination' % bin_pid)
except KeyboardInterrupt:
selflog('KeyboardInterrupt, exiting')
return DIE
except JobExit:
pass
finally:
try:
shutil.rmtree(job_dir)
selflog('DELETED temporary directory %s for %s' % (job_dir, bin_pid))
except:
selflog('WARNING cannot remove temporary directory %s for %s' % (job_dir, bin_pid))
try:
shutil.rmtree(zip_dir)
selflog('DELETED temporary directory %s for %s' % (zip_dir, bin_pid))
except:
selflog('WARNING cannot remove temporary directory %s for %s' % (zip_dir, bin_pid))
selflog('DONE - no more actions for %s' % bin_pid)
def generate_ids(n,ns=''):
return json.dumps([ns + gen_id() for _ in range(n)])
def new_process_id(self):
try:
return gen_id(self.config.namespace)
except KeyError:
return gen_id()
def new_process_id(self):
try:
return gen_id(self.config.namespace)
except KeyError:
return gen_id()
return (connection,cursor)
