def load( self, pathname ):
timer.start('client.import')
with open(pathname, 'r') as f:
line = f.readline()
cnt = 0
while line:
if (cnt%1000)==0:
sys.stdout.write(str(cnt))
sys.stdout.write('.')
sys.stdout.flush()
obj = jsonlib.loads( line )
if 'size' in obj and 'body' not in obj:
pathname = glob.tmp_file_directory+uuid()
with open(pathname,'w') as tf:
remaining = obj['size']
while remaining>(1024*1024):
sys.stdout.write(',')
sys.stdout.flush()
tf.write( f.read(1024*1024) )
remaining -= (1024*1024)
tf.write( f.read(remaining) )
item = Item( obj, path=pathname )
else:
item = Item( obj )
glob.db.insert( item )
if item.type=='call':
glob.db.task_add( item )
cnt += 1
line = f.readline()
diff = timer.stop('client.import')
print 'Imported in:',diff
def envi_add( self, **kwargs ):
timer.start('client.envi_add')
if 'engine' in kwargs:
if kwargs['engine'] == 'wrapper':
obj = {'engine':kwargs['engine'],
'open':kwargs['open'], 'close':kwargs['close'],
'args':kwargs['args'], 'params':kwargs['params']}
it = Item( type='envi', body=obj )
glob.db.insert(it)
return it.cbid
elif kwargs['engine'] == 'umbrella':
filename = kwargs['spec']
cbid = self.file_add( filename )
obj = {'engine':kwargs['engine'], 'spec':cbid,
'cvmfs_http_proxy':kwargs['cvmfs_http_proxy'],
'sandbox_mode':kwargs['sandbox_mode'],
'log':kwargs['log'], 'args':[cbid]}
it = Item( type='envi', body=obj )
glob.db.insert(it)
return it.cbid
else:
timer.stop('client.envi_add')
self.op_cnt += 1
return self.nil
else:
timer.stop('client.envi_add')
self.op_cnt += 1
return self.nil
def data_add( self, data ):
timer.start('client.data_add')
it = Item( type='file', body=data )
glob.db.insert(it)
timer.stop('client.data_add')
self.op_cnt += 1
return it.cbid
def envi_add( self, **kwargs ):
timer.start('client.envi_add')
if 'engine' in kwargs:
if kwargs['engine'] == 'wrapper':
obj = {'engine':kwargs['engine'],
'open':kwargs['open'], 'close':kwargs['close'],
'args':kwargs['args'], 'params':kwargs['params']}
if 'http_proxy' in kwargs:
obj['http_proxy'] = kwargs['http_proxy']
it = Item( type='envi', body=obj )
glob.db.insert(it)
return it.cbid
elif kwargs['engine'] == 'umbrella':
filename = kwargs['spec']
cbid = self.file_add( filename )
args = [cbid]
params = ['SPEC.umbrella']
if kwargs['cms_siteconf']:
filename2 = kwargs['cms_siteconf']
cbid2 = self.file_add( filename2 )
args.append(cbid2)
params.append(filename2)
obj = {'engine':kwargs['engine'], 'spec':cbid,
'cvmfs_http_proxy':kwargs['cvmfs_http_proxy'],
'cms_siteconf':kwargs['cms_siteconf'],
'sandbox_mode':kwargs['sandbox_mode'],
'log':kwargs['log'], 'args':args, 'params':params}
it = Item( type='envi', body=obj )
glob.db.insert(it)
return it.cbid
else:
timer.stop('client.envi_add')
self.op_cnt += 1
return self.nil
else:
timer.stop('client.envi_add')
self.op_cnt += 1
return self.nil
def file_add( self, filename ):
timer.start('client.file_add')
#start = time.time()
it = Item( type='file', path=filename, new_path=glob.tmp_file_directory+uuid() )
#elapsed = time.time()-start
glob.db.insert(it)
timer.stop('client.file_add')
self.op_cnt += 1
return it.cbid
def task_add( self, returns, env, cmd, args=[], params=[], types=[], env_vars={}, precise=True):
timer.start('client.task_add')
obj = {'returns':returns, 'env':env, 'cmd':cmd, 'args':args, 'params':params, 'types':types, 'env_vars':env_vars, 'precise':precise}
it = Item( type='call', body=obj )
glob.db.insert( it )
glob.db.task_add( it )
results = []
for i in range( 0, len(returns) ):
results.append(it.cbid+':'+str(i))
timer.stop('client.task_add')
self.op_cnt += 1
#if (self.op_cnt%100)==0:
# time.sleep(0.1)
return results
def finish(self):
call_body = self.call.body
if self.call.step:
glob.workflow_id = self.call.wfid
glob.workflow_step = self.call.step
full_content = True
results = []
sizes = []
timer.start('work.stage_out')
if os.path.isfile(self.sandbox + self.task_dbg_log):
with open(self.sandbox + self.task_dbg_log) as f:
for line in f:
if line.startswith('----prune_task_return'):
(a, fname, size, cbid) = line[:-1].split('|')
if size and len(size) > 0:
sizes.append(int(size))
else:
sizes.append(-1)
results.append(cbid)
elif line.startswith('----prune_task_start'):
(a, taskstart) = line[:-1].split('|')
elif line.startswith('----prune_task_end'):
(a, taskfinish) = line[:-1].split('|')
if os.path.isfile(self.sandbox + self.env_dbg_log):
with open(self.sandbox + self.env_dbg_log) as f:
for line in f:
if line.startswith('----prune_env_start'):
(a, envstart) = line[:-1].split('|')
elif line.startswith('----prune_env_end'):
(a, envfinish) = line[:-1].split('|')
if os.path.isfile(self.sandbox + self.cmd_dbg_log):
with open(self.sandbox + self.cmd_dbg_log) as f:
for line in f:
if line.startswith('----prune_cmd_start'):
(a, cmdstart) = line[:-1].split('|')
elif line.startswith('----prune_cmd_end'):
(a, cmdfinish) = line[:-1].split('|')
for i, src in enumerate(call_body['returns']):
if os.path.isfile(self.sandbox + src):
it = Item(type='temp',
cbid=results[i],
dbid=self.call.cbid + ':' + str(i),
path=self.sandbox + src,
size=sizes[i])
glob.db.insert(it)
else:
d('exec',
'sandbox return file not found: ' + self.sandbox + src)
full_content = False
timer.stop('work.stage_out')
if full_content:
d('exec', 'Sandbox success kept at: ' + self.sandbox)
timer.start('work.report')
cmd_time = float(cmdfinish) - float(cmdstart)
env_time = (float(envfinish) - float(envstart)) - cmd_time
task_time = (float(taskfinish) -
float(taskstart)) - cmd_time - env_time
meta = {
'task_time': task_time,
'env_time': env_time,
'cmd_time': cmd_time
}
it = Item(type='work',
cbid=self.call.cbid + '()',
meta=meta,
body={
'results': results,
'sizes': sizes
})
glob.db.insert(it)
timer.add(cmd_time, 'work.execution_time')
timer.add(env_time, 'work.environment_overhead')
timer.add(task_time, 'work.prune_overhead')
timer.stop('work.report')
else:
self.results = results
d('exec', 'sandbox failure kept: ' + self.sandbox)
from class_item import Item
from class_populacao import Populacao
populacao = Populacao()
item1 = Item('item 1', 3, 40, 3)
item2 = Item('item 2', 2, 100, 2)
item3 = Item('item 3', 3, 50, 5)
#criando populacao inicial
populacao.gerarPopulcaoInicial(item1, item2, item3)
while populacao.getParada() < 50:
populacao.gerarNovaPopulacao()
def create_item_at(self, item_id, pos):
tmp_item = Item(item_id, pos)
self.item_list.append(tmp_item)
return tmp_item
def createItemFromId(p_itemId):
db = doSql()
if not p_itemId:
return None
query = "SELECT * FROM get_item("+str(p_itemId)+");"
((
id,
name,
iType,
price,
imgurl
),) = db.execqry(query, False)
item = Item()
item.setId(id)
item.setName(name)
item.setType(iType)
item.setPrice(price)
item.setImg(imgurl)
return item
def createItem(p_itemName,p_itemType,p_itemPrice,p_itemImg):
db = doSql()
query = "SELECT * FROM add_item('"+\
p_itemPrice+"','"+\
p_itemName+"','"+\
p_itemType+"','"+\
p_itemImg+"');"
item = Item()
item.setId(0)
item.setName(p_itemName)
item.setType(p_itemType)
item.setPrice(p_itemPrice)
item.setImg(p_itemImg)
((status_code,
),) = db.execqry(query, True)
return item
def finish(self):
call_body = self.call.body
if self.call.step:
glob.workflow_id = self.call.wfid
glob.workflow_step = self.call.step
full_content = True
results = []
sizes = []
timer.start('work.stage_out')
with open(self.sandbox + 'PRUNE_TASK_LOG') as f:
for line in f:
if line.startswith('sha1sum'):
(a, fname, cbid) = line.split(' ')
results.append(cbid.strip())
elif line.startswith('filesize'):
(a, fname, size) = line.split(' ')
sizes.append(int(size.strip()))
elif line.startswith('prune_call_start'):
(a, estart) = line.split(' ')
elif line.startswith('prune_call_end'):
(a, efinish) = line.split(' ')
elif line.startswith('prune_cmd_start'):
(a, cstart) = line.split(' ')
elif line.startswith('prune_cmd_end'):
(a, cfinish) = line.split(' ')
elif line.startswith('prune_stage_out_start'):
(a, sostart) = line.split(' ')
elif line.startswith('prune_stage_out_end'):
(a, sofinish) = line.split(' ')
if self.env and self.env.body['engine'] == 'umbrella':
for i, src in enumerate(call_body['returns']):
full_src = "/tmp/umbrella_prune/%s/%s" % (self.sbid, src)
if os.path.isfile(full_src):
it = Item(type='temp',
cbid=results[i],
dbid=self.call.cbid + ':' + str(i),
path=full_src,
size=sizes[i])
glob.db.insert(it)
else:
d('exec', 'sandbox return file not found: ' + full_src)
full_content = False
else:
for i, src in enumerate(call_body['returns']):
if os.path.isfile(self.sandbox + src):
it = Item(type='temp',
cbid=results[i],
dbid=self.call.cbid + ':' + str(i),
path=self.sandbox + src,
size=sizes[i])
glob.db.insert(it)
else:
d('exec',
'sandbox return file not found: ' + self.sandbox + src)
full_content = False
timer.stop('work.stage_out')
if full_content:
d('exec', 'sandbox kept at: ' + self.sandbox)
timer.start('work.report')
wall_time = float(cfinish) - float(cstart)
env_time = (float(efinish) - float(estart)) - wall_time
meta = {'wall_time': wall_time, 'env_time': env_time}
it = Item(type='work',
cbid=self.call.cbid + '()',
meta=meta,
body={
'results': results,
'sizes': sizes
})
glob.db.insert(it)
timer.add(env_time, 'work.environment_use')
timer.add(wall_time, 'work.wall_time')
timer.add((float(sofinish) - float(sostart)),
'work.stage_out.hash')
timer.stop('work.report')
else:
d('exec', 'sandbox kept: ' + self.sandbox)
def finish(self):
call_body = self.call.body
full_content = True
meta_results = [] # place for debugging information (for example)
tmp_results = []
tmp_sizes = []
try:
print 'Prune Task stdout:', self.task.output
with open(self.sandbox + 'PRUNE_TASK_LOG') as f:
for line in f:
#print line
if line.startswith('sha1sum'):
(a, fname, cbid) = line.split(' ')
tmp_results.append(cbid.strip())
elif line.startswith('filesize'):
(a, fname, size) = line.split(' ')
tmp_sizes.append(int(size.strip()))
elif line.startswith('prune_call_start'):
(a, estart) = line.split(' ')
elif line.startswith('prune_call_end'):
(a, efinish) = line.split(' ')
elif line.startswith('prune_cmd_start'):
(a, cstart) = line.split(' ')
elif line.startswith('prune_cmd_end'):
(a, cfinish) = line.split(' ')
elif line.startswith('prune_stage_out_start'):
(a, sostart) = line.split(' ')
elif line.startswith('prune_stage_out_end'):
(a, sofinish) = line.split(' ')
for i, rtrn in enumerate(call_body['returns']):
result = self.results[i]
print result, rtrn
if os.path.isfile(result):
it = Item(type='temp',
cbid=tmp_results[i],
dbid=self.call.cbid + ':' + str(i),
path=result,
size=tmp_sizes[i])
glob.db.insert(it)
else:
d('exec', 'sandbox return file not found: ', rtrn)
full_content = False
if full_content:
d('exec', 'sandbox kept at: ' + self.sandbox)
timer.start('work.report')
wall_time = float(cfinish) - float(cstart)
env_time = (float(efinish) - float(estart)) - wall_time
meta = {'wall_time': wall_time, 'env_time': env_time}
it = Item(type='work',
cbid=self.call.cbid + '()',
meta=meta,
body={
'results': tmp_results,
'sizes': tmp_sizes
})
glob.db.insert(it)
timer.add(env_time, 'work.environment_use')
timer.add(wall_time, 'work.wall_time')
timer.add((float(sofinish) - float(sostart)),
'work.stage_out.hash')
timer.stop('work.report')
else:
d('exec', 'sandbox kept: ' + self.sandbox)
except:
print traceback.format_exc()
glob.db.task_fail(self.call)
