def execute(job_obj, image_path=None, source_type=None):
try:
directory = os.path.join(job_obj.storage_path, str(job_obj.jobid))
if not os.path.exists(directory):
os.makedirs(directory)
os.chmod(directory, 0776)
result_path = directory
# Run an executable as defined by the user
if job_obj.url is None:
job_obj.url = os.path.join(conf.PIC_URL, job_obj.userid, job_obj.jobid)
log_to_terminal('Files Stored. Waiting for one of the worker nodes to handle this request', job_obj.socketid)
parsed_dict = {'jobid': job_obj.jobid, 'userid': job_obj.userid,
'image_path': str(image_path), 'result_path': str(result_path),
'dir': str(directory), 'url': job_obj.url,
'source_type': source_type, 'exec': job_obj.executable,
'count': job_obj.count, 'token': job_obj.token,
'socketid': job_obj.socketid, 'params': job_obj.params,
'dropbox_token': job_obj.dropbox_token}
r.publish('chat', json.dumps({'jobinfo': parsed_dict, 'socketid': str(job_obj.socketid)}))
run.delay(parsed_dict)
except:
log_and_exit(str(traceback.format_exc()), job_obj.socketid)
return str('Error pushing to the job queue')
return str(parsed_dict)
def saveFilesAndProcess(request, job_obj):
"""
TODO: Specify four types of location
1.) CloudCV General Dataset
2.) CloudCV me folder, me folder represents the user folder
3.) Dropbox folder
4.) Local System - Either save it as a test and delete, or permanently by specifying a name
"""
# If the server is different. The storage path needs to be changed to shared folders.
parsed_params = parseParameters(job_obj.params)
if 'server' in parsed_params and parsed_params['server'] == 'decaf_server':
job_obj.storage_path = '/srv/share/cloudcv/jobs/' + job_obj.userid + '/'
if not os.path.exists(job_obj.storage_path):
os.mkdir(job_obj.storage_path)
os.chmod(job_obj.storage_path, 0775)
job_obj.url = 'http://godel.ece.vt.edu/cloudcv/fileupload/media/pictures/decaf_server/'+ job_obj.userid + \
'/' + job_obj.jobid
r.publish('chat', json.dumps({'error': str('Special Server Identified'), 'socketid': job_obj.socketid}))
# Save files either through dropbox or client local system for use an throw scenario
if job_obj.dropbox_path is not None:
result = saveDropboxFiles.delay(job_obj.__dict__)
return 'Downloading content from dropbox. Execution will begin after downloading finishes'
else:
files_all = getFilesFromRequest(request, job_obj.count)
if len(files_all) == 0:
return 'Length of files = 0'
if len(files_all) > 50:
r.publish('chat', json.dumps({'error': str('Shutting down now.'),
'socketid':job_obj.socketid, 'end': 'yes'}))
return 'Length of files higher than the limit of 50. Please use dropbox'
job_directory = os.path.join(job_obj.storage_path, str(job_obj.jobid))
log_to_terminal('Processing files', job_obj.socketid)
for single_file in files_all:
try:
# new_file_name = saveInPictureDatabase(single_file)
path = conf.PIC_DIR
size = (500, 500)
resizeImageAndTransfer(path, job_directory, size, single_file)
except Exception as e:
print str(traceback.format_exc())
raise e
response = core_execute.execute(job_obj, job_directory, 'local')
return response
def decafImages(src_path, output_path, socketid, result_path, single_file_name=''):
try:
#Entire Directory
if os.path.isdir(os.path.join(src_path,single_file_name)):
for file_name in os.listdir(src_path):
tags = {}
image_path = os.path.join(src_path, file_name)
if os.path.isfile(image_path):
""" Trying to get the output of classify python script to send to user - Part 1/4
myPrint = CustomPrint(socketid)
old_stdout=sys.stdout
sys.stdout = myPrint
"""
print 'Running caffe classify on multiple images'
mat_file_path = decaf.calculate_decaf_image(file_name, src_path, output_path, 3, socketid, tags)
print tags
""" Part 2/2
sys.stdout=old_stdout
"""
log_to_terminal("Results: "+str(tags), socketid)
# sorted_tags = sorted(tags.iteritems(), key=operator.itemgetter(1), reverse=True)
# webResult = {}
#webResult[str(result_path + file_name)] = sorted_tags
result_url = urlparse(result_path).path
r.publish('chat',
json.dumps({'web_result': os.path.join(result_url, 'results', file_name+'.mat'), 'socketid': str(socketid)}))
log_to_terminal('Thank you for using CloudCV', socketid)
# Single File
else:
""" Part 3/4
myPrint = CustomPrint(socketid)
old_stdout=sys.stdout
sys.stdout = myPrint
"""
tags = {}
print 'Running caffe classify on a single image: ' + single_file_name
mat_file_path = decaf.calculate_decaf_image(single_file_name, src_path, output_path, 3, socketid, tags)
""" Part 4/4
sys.stdout=old_stdout
"""
log_to_terminal("Results: "+str(tags), socketid)
# tags = sorted(tags.iteritems(), key=operator.itemgetter(1), reverse=True)
# web_result = {}
# web_result[str(result_path)] = tags
result_url = os.path.dirname(urlparse(result_path).path)
r.publish('chat', json.dumps({'web_result': os.path.join(result_url, 'results', single_file_name+'.mat'), 'socketid': str(socketid)}))
log_to_terminal('Thank you for using CloudCV', socketid)
except Exception as e:
log_to_terminal(str(traceback.format_exc()), socketid)
def classifyImages(src_path, socketid, result_path):
try:
#Entire Directory
if os.path.isdir(src_path):
for file_name in os.listdir(src_path):
image_path = os.path.join(src_path, file_name)
if os.path.isfile(image_path):
""" Trying to get the output of classify python script to send to user - Part 1/4
myPrint = CustomPrint(socketid)
old_stdout=sys.stdout
sys.stdout = myPrint
"""
print 'Running caffe classify...'
tags = caffe_classify_image(image_path)
""" Part 2/2
sys.stdout=old_stdout
"""
log_to_terminal("Results: "+str(tags), socketid)
# tags = sorted(tags.iteritems(), key=operator.itemgetter(1),reverse=True)
webResult = {}
webResult[str(os.path.join(result_path, file_name))] = tags
r.publish('chat',
json.dumps({'web_result': json.dumps(webResult), 'socketid': str(socketid)}))
log_to_terminal('Thank you for using CloudCV', socketid)
# Single File
else:
""" Part 3/4
myPrint = CustomPrint(socketid)
old_stdout=sys.stdout
sys.stdout = myPrint
"""
print 'Running caffe classify...'
tags = caffe_classify_image(src_path)
""" Part 4/4
sys.stdout=old_stdout
"""
log_to_terminal("Results: "+str(tags), socketid)
# tags = sorted(tags.iteritems(), key=operator.itemgetter(1), reverse=True)
web_result = {}
web_result[str(result_path)] = tags
r.publish('chat', json.dumps({'web_result': json.dumps(web_result), 'socketid': str(socketid)}))
log_to_terminal('Thank you for using CloudCV', socketid)
except Exception as e:
log_to_terminal(str(traceback.format_exc()), socketid)
def downloadFiles(job_dict):
log_to_terminal('Dropbox Queue Handler: Downloading images', job_dict['socketid'])
client = dropbox.client.DropboxClient(job_dict['dropbox_token'])
try:
folder_metadata = client.metadata(job_dict['dropbox_path'])
except Exception as e:
log_and_exit('Path: ' + job_dict['dropbox_path'] + ' not found', job_dict['socketid'])
return
log(job_dict['dropbox_path'], downloadFiles.__name__)
log(job_dict['dropbox_token'], downloadFiles.__name__)
for content in folder_metadata['contents']:
if content['is_dir'] is False and 'image' in str(content['mime_type']):
file, file_metadata = client.get_file_and_metadata(content['path'])
rel_path = ntpath.dirname(str(content['path'])).lstrip('/')
log_to_terminal(os.path.join(job_dict['storage_path'] ,rel_path , ntpath.basename(content['path'])),
job_dict['socketid'])
if not os.path.exists(os.path.join(job_dict['storage_path'], rel_path)):
os.makedirs(os.path.join(job_dict['storage_path'], rel_path))
os.chmod(os.path.join(job_dict['storage_path'] , rel_path), 0775)
img = Image.open(StringIO(file.read()))
img.save(os.path.join(job_dict['storage_path'], rel_path,
ntpath.basename(content['path'])),
file_metadata['mime_type'].split('/')[1])
try:
job_obj = Job()
for k,v in job_dict.iteritems():
setattr(job_obj, k, v)
log(str(job_dict['storage_path'] + job_dict['dropbox_path'].strip('/')), 'directory')
if job_obj.count is None or int(job_obj.count) <= 0:
count = 0
for name in os.listdir(os.path.join(job_dict['storage_path'], job_dict['dropbox_path'].strip('/'))):
log(str(name), 'name')
if os.path.isfile(os.path.join(job_dict['storage_path'], job_dict['dropbox_path'].strip('/'), name)):
count +=1
log(str(name), 'name')
job_obj.count = count
# log_to_terminal('JobID: ' + str(job_obj.jobid), job_obj.socketid)
# log_to_terminal('Count: ' + str(job_obj.count), job_obj.socketid)
core_execute.execute(job_obj, os.path.join(job_dict['storage_path'], job_dict['dropbox_path'].strip('/')), 'dropbox')
except Exception as e:
print str(traceback.format_exc())
log_and_exit(str(traceback.format_exc()), job_dict['socketid'])
def classifyImages(src_path, socketid, result_path):
try:
#Entire Directory
if os.path.isdir(src_path):
for file_name in os.listdir(src_path):
image_path = os.path.join(src_path, file_name)
if os.path.isfile(image_path):
print 'Running caffe classify...'
tags = default_classify.caffe_classify_image(image_path)
log_to_terminal("Results: "+str(tags), socketid)
# tags = sorted(tags.iteritems(), key=operator.itemgetter(1),reverse=True)
webResult = {}
webResult[str(os.path.join(result_path, file_name))] = tags
r.publish('chat',
json.dumps({'web_result': json.dumps(webResult), 'socketid': str(socketid)}))
log_to_terminal('Thank you for using CloudCV', socketid)
# Single File
else:
print 'Running caffe classify...'
tags = default_classify.caffe_classify_image(src_path)
log_to_terminal("Results: "+str(tags), socketid)
# tags = sorted(tags.iteritems(), key=operator.itemgetter(1), reverse=True)
web_result = {}
web_result[str(result_path)] = tags
r.publish('chat', json.dumps({'web_result': json.dumps(web_result), 'socketid': str(socketid)}))
log_to_terminal('Thank you for using CloudCV', socketid)
except Exception as e:
log_to_terminal(str(traceback.format_exc()), socketid)
def classifyImages(src_path, socketid, result_path):
try:
#Entire Directory
if os.path.isdir(src_path):
for file_name in os.listdir(src_path):
image_path = os.path.join(src_path, file_name)
if os.path.isfile(image_path):
print 'Running caffe classify...'
tags = default_classify.caffe_classify_image(image_path)
log_to_terminal("Results: "+str(tags), socketid)
# tags = sorted(tags.iteritems(), key=operator.itemgetter(1),reverse=True)
webResult = {}
webResult[str(os.path.join(result_path, file_name))] = tags
r.publish('chat',
json.dumps({'web_result': json.dumps(webResult), 'socketid': str(socketid)}))
log_to_terminal('Thank you for using CloudCV', socketid)
# Single File
else:
print 'Running caffe classify...'
tags = default_classify.caffe_classify_image(src_path)
log_to_terminal("Results: "+str(tags), socketid)
# tags = sorted(tags.iteritems(), key=operator.itemgetter(1), reverse=True)
web_result = {}
web_result[str(result_path)] = tags
r.publish('chat', json.dumps({'web_result': json.dumps(web_result), 'socketid': str(socketid)}))
log_to_terminal('Thank you for using CloudCV', socketid)
except Exception as e:
log_to_terminal(str(traceback.format_exc()), socketid)
def runImageStitching(list, result_path, socketid):
try:
popen = subprocess.Popen(list,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
count = 1
print 'Coming Here'
while True:
popen.poll()
if (popen.stdout):
line = popen.stdout.readline()
popen.stdout.flush()
if (popen.stderr):
errline = popen.stderr.readline()
popen.stderr.flush()
# r = redis.StrictRedis(host = '127.0.0.1' , port=6379, db=0)
if line:
# r = redis.StrictRedis(host='127.0.0.1', port=6379, db=0)
log_to_terminal(line, str(socketid))
# fi.write(line+'*!*'+socketid+'\n')
print count, line, '\n'
count += 1
# time.sleep(1)
if errline:
# r = redis.StrictRedis(host='127.0.0.1', port=6379, db=0)
log_to_terminal(errline, str(socketid))
# fi.write(line+'*!*'+socketid+'\n')
print count, line, '\n'
count += 1
if line == '':
break
log_to_terminal('Thank you for using CloudCV', str(socketid))
r.publish(
'chat',
json.dumps({
'web_result': result_path,
'socketid': str(socketid)
}))
except Exception as e:
log_to_terminal(str(traceback.format_exc()), str(socketid))
print str(traceback.format_exc())
return '\n', '\n'
def classifyImagesWithNewModel(jobPath, socketid, result_path):
print jobPath, socketid, result_path
try:
ImagePath = os.path.join(jobPath, 'test')
modelPath = os.path.join(jobPath, 'util')
new_labels = sio.loadmat(os.path.join(modelPath, 'new_labels.mat'))
new_labels_cells = new_labels['WNID']
# Set the right path to your model file, pretrained model,
# and the image you would like to classify.
MODEL_FILE = os.path.join(modelPath, 'newCaffeModel.prototxt')
PRETRAINED = os.path.join(modelPath, 'newCaffeModel.caffemodel')
caffe.set_phase_test()
caffe.set_mode_gpu()
net = caffe.Classifier(
MODEL_FILE,
PRETRAINED,
mean=np.load(
os.path.join(CAFFE_DIR,
'python/caffe/imagenet/ilsvrc_2012_mean.npy')),
channel_swap=(2, 1, 0),
raw_scale=255,
image_dims=(256, 256))
results = {}
if os.path.isdir(ImagePath):
for file_name in os.listdir(ImagePath):
image_path = os.path.join(ImagePath, file_name)
if os.path.isfile(image_path):
tags = caffe_classify_image(net, image_path,
new_labels_cells)
log_to_terminal("Results: " + str(tags), socketid)
webResult = {}
webResult[os.path.join(result_path, file_name)] = tags
r.publish(
'chat',
json.dumps({
'web_result': json.dumps(webResult),
'socketid': str(socketid)
}))
log_to_terminal('Thank you for using CloudCV', socketid)
except Exception as e:
log_to_terminal(str(traceback.format_exc()), socketid)
def runImageStitching(list, result_path, socketid):
try:
popen = subprocess.Popen(list,stdout=subprocess.PIPE, stderr=subprocess.PIPE)
count=1
print 'Coming Here'
while True:
popen.poll()
if(popen.stdout):
line=popen.stdout.readline()
popen.stdout.flush()
if(popen.stderr):
errline = popen.stderr.readline()
popen.stderr.flush()
# r = redis.StrictRedis(host = '127.0.0.1' , port=6379, db=0)
if line:
# r = redis.StrictRedis(host='127.0.0.1', port=6379, db=0)
log_to_terminal(line, str(socketid))
# fi.write(line+'*!*'+socketid+'\n')
print count,line, '\n'
count += 1
# time.sleep(1)
if errline:
# r = redis.StrictRedis(host='127.0.0.1', port=6379, db=0)
log_to_terminal(errline, str(socketid))
# fi.write(line+'*!*'+socketid+'\n')
print count,line, '\n'
count += 1
if line == '':
break
log_to_terminal('Thank you for using CloudCV', str(socketid))
r.publish('chat', json.dumps({'web_result': result_path, 'socketid': str(socketid)}))
except Exception as e:
log_to_terminal(str(traceback.format_exc()), str(socketid))
print str(traceback.format_exc())
return '\n', '\n'
def answerQuestion(feat_path, question, socketid, imageid):
try:
print 'Thinking...'
# For now, using numpy archives
feat_path = feat_path + '.npy'
ans = vqa_answer(feat_path, question)
log_to_terminal("Answer found: " + ans, socketid)
# tags = sorted(tags.iteritems(), key=operator.itemgetter(1), reverse=True)
web_result = {}
web_result[imageid] = [ans]
r.publish('chat', json.dumps({'web_result': json.dumps(web_result), 'socketid': str(socketid)}))
log_to_terminal('Thank you for using CloudCV', socketid)
except Exception as e:
log_to_terminal(str(traceback.format_exc()), socketid)
def classifyImagesWithNewModel(jobPath, socketid, result_path):
print jobPath, socketid, result_path
try:
ImagePath = os.path.join(jobPath,'test')
modelPath = os.path.join(jobPath,'util')
new_labels = sio.loadmat(os.path.join(modelPath,'new_labels.mat'))
new_labels_cells = new_labels['WNID']
# Set the right path to your model file, pretrained model,
# and the image you would like to classify.
MODEL_FILE = os.path.join(modelPath,'newCaffeModel.prototxt')
PRETRAINED = os.path.join(modelPath,'newCaffeModel.caffemodel')
caffe.set_phase_test()
caffe.set_mode_gpu()
net = caffe.Classifier(MODEL_FILE, PRETRAINED,
mean=np.load(os.path.join(CAFFE_DIR, 'python/caffe/imagenet/ilsvrc_2012_mean.npy')),
channel_swap=(2, 1, 0),
raw_scale=255,
image_dims=(256, 256))
results = {}
if os.path.isdir(ImagePath):
for file_name in os.listdir(ImagePath):
image_path = os.path.join(ImagePath, file_name)
if os.path.isfile(image_path):
tags = caffe_classify_image(net, image_path, new_labels_cells)
log_to_terminal("Results: "+str(tags), socketid)
webResult = {}
webResult[os.path.join(result_path,file_name)] = tags
r.publish('chat',
json.dumps({'web_result': json.dumps(webResult), 'socketid': str(socketid)}))
log_to_terminal('Thank you for using CloudCV', socketid)
except Exception as e:
log_to_terminal(str(traceback.format_exc()), socketid)
def answerQuestion(feat_path, question, socketid, imageid):
try:
print 'Thinking...'
# For now, using numpy archives
feat_path = feat_path + '.npy'
ans = vqa_answer(feat_path, question)
log_to_terminal("Answer found: " + ans, socketid)
# tags = sorted(tags.iteritems(), key=operator.itemgetter(1), reverse=True)
web_result = {}
web_result[imageid] = [ans]
r.publish(
'chat',
json.dumps({
'web_result': json.dumps(web_result),
'socketid': str(socketid)
}))
log_to_terminal('Thank you for using CloudCV', socketid)
except Exception as e:
log_to_terminal(str(traceback.format_exc()), socketid)
def decafImages(src_path,
output_path,
socketid,
result_path,
single_file_name=''):
try:
#Entire Directory
if os.path.isdir(os.path.join(src_path, single_file_name)):
for file_name in os.listdir(src_path):
tags = {}
image_path = os.path.join(src_path, file_name)
if os.path.isfile(image_path):
""" Trying to get the output of classify python script to send to user - Part 1/4
myPrint = CustomPrint(socketid)
old_stdout=sys.stdout
sys.stdout = myPrint
"""
print 'Running caffe classify on multiple images'
mat_file_path = decaf.calculate_decaf_image(
file_name, src_path, output_path, 3, socketid, tags)
print tags
""" Part 2/2
sys.stdout=old_stdout
"""
log_to_terminal("Results: " + str(tags), socketid)
# sorted_tags = sorted(tags.iteritems(), key=operator.itemgetter(1), reverse=True)
# webResult = {}
#webResult[str(result_path + file_name)] = sorted_tags
result_url = urlparse(result_path).path
r.publish(
'chat',
json.dumps({
'web_result':
os.path.join(result_url, 'results',
file_name + '.mat'),
'socketid':
str(socketid)
}))
log_to_terminal('Thank you for using CloudCV', socketid)
# Single File
else:
""" Part 3/4
myPrint = CustomPrint(socketid)
old_stdout=sys.stdout
sys.stdout = myPrint
"""
tags = {}
print 'Running caffe classify on a single image: ' + single_file_name
mat_file_path = decaf.calculate_decaf_image(
single_file_name, src_path, output_path, 3, socketid, tags)
""" Part 4/4
sys.stdout=old_stdout
"""
log_to_terminal("Results: " + str(tags), socketid)
# tags = sorted(tags.iteritems(), key=operator.itemgetter(1), reverse=True)
# web_result = {}
# web_result[str(result_path)] = tags
result_url = os.path.dirname(urlparse(result_path).path)
r.publish(
'chat',
json.dumps({
'web_result':
os.path.join(result_url, 'results',
single_file_name + '.mat'),
'socketid':
str(socketid)
}))
log_to_terminal('Thank you for using CloudCV', socketid)
except Exception as e:
log_to_terminal(str(traceback.format_exc()), socketid)
def decafImages(src_path, output_path, socketid, result_path, single_file_name='', modelname=''):
if modelname is not '':
lMODEL_FILE = str(os.path.join(conf.CAFFE_DIR, 'models',modelname,'deploy.prototxt'))
lPRETRAINED = str(os.path.join(conf.CAFFE_DIR, 'models', modelname, modelname+'.caffemodel'))
r.publish('chat', json.dumps({'error': lMODEL_FILE+' '+lPRETRAINED, 'socketid': socketid}))
caffe.set_phase_test()
caffe.set_mode_cpu()
modelnet = caffe.Classifier(lMODEL_FILE, lPRETRAINED)
#r.publish('chat', json.dumps({'error': str(modelname), 'socketid': socketid}))
else:
modelnet = None
try:
#Entire Directory
if os.path.isdir(os.path.join(src_path,single_file_name)):
for file_name in os.listdir(src_path):
tags = {}
image_path = os.path.join(src_path, file_name)
if os.path.isfile(image_path):
""" Trying to get the output of classify python script to send to user - Part 1/4
myPrint = CustomPrint(socketid)
old_stdout=sys.stdout
sys.stdout = myPrint
"""
print 'Running caffe classify on multiple images'
mat_file_path = decaf.calculate_decaf_image(file_name, src_path, output_path, 3, socketid, tags, modelname, modelnet)
print tags
""" Part 2/2
sys.stdout=old_stdout
"""
log_to_terminal("Results: "+str(tags), socketid)
# sorted_tags = sorted(tags.iteritems(), key=operator.itemgetter(1), reverse=True)
# webResult = {}
#webResult[str(result_path + file_name)] = sorted_tags
result_url = urlparse(result_path).path
r.publish('chat',
json.dumps({'web_result': os.path.join(result_url, 'results', file_name+'.mat'), 'socketid': str(socketid)}))
log_to_terminal('Thank you for using CloudCV', socketid)
# Single File
else:
""" Part 3/4
myPrint = CustomPrint(socketid)
old_stdout=sys.stdout
sys.stdout = myPrint
"""
tags = {}
print 'Running caffe classify on a single image: ' + single_file_name
try:
mat_file_path = decaf.calculate_decaf_image(single_file_name, src_path, output_path, 3, socketid, tags, modelname, modelnet)
except Exception as e:
print str(e)
""" Part 4/4
sys.stdout=old_stdout
"""
log_to_terminal("Results: "+str(tags), socketid)
# tags = sorted(tags.iteritems(), key=operator.itemgetter(1), reverse=True)
# web_result = {}
# web_result[str(result_path)] = tags
result_url = os.path.dirname(urlparse(result_path).path)
r.publish('chat', json.dumps({'web_result': os.path.join(result_url, 'results', single_file_name+'.mat'), 'socketid': str(socketid)}))
log_to_terminal('Thank you for using CloudCV', socketid)
except Exception as e:
log_to_terminal(str(traceback.format_exc()), socketid)
def trainModel(save_dir, socketid):
train_fast.modelUpdate(save_dir+'/')
log_to_terminal('Finished training your model with the new categories. Now, upload some test images to test this model. ', socketid)
def decafImages(src_path,
output_path,
socketid,
result_path,
single_file_name='',
modelname=''):
if modelname is not '':
lMODEL_FILE = str(
os.path.join(conf.CAFFE_DIR, 'models', modelname,
'deploy.prototxt'))
lPRETRAINED = str(
os.path.join(conf.CAFFE_DIR, 'models', modelname,
modelname + '.caffemodel'))
r.publish(
'chat',
json.dumps({
'error': lMODEL_FILE + ' ' + lPRETRAINED,
'socketid': socketid
}))
caffe.set_phase_test()
caffe.set_mode_cpu()
modelnet = caffe.Classifier(lMODEL_FILE, lPRETRAINED)
#r.publish('chat', json.dumps({'error': str(modelname), 'socketid': socketid}))
else:
modelnet = None
try:
#Entire Directory
if os.path.isdir(os.path.join(src_path, single_file_name)):
for file_name in os.listdir(src_path):
tags = {}
image_path = os.path.join(src_path, file_name)
if os.path.isfile(image_path):
""" Trying to get the output of classify python script to send to user - Part 1/4
myPrint = CustomPrint(socketid)
old_stdout=sys.stdout
sys.stdout = myPrint
"""
print 'Running caffe classify on multiple images'
mat_file_path = decaf.calculate_decaf_image(
file_name, src_path, output_path, 3, socketid, tags,
modelname, modelnet)
print tags
""" Part 2/2
sys.stdout=old_stdout
"""
log_to_terminal("Results: " + str(tags), socketid)
# sorted_tags = sorted(tags.iteritems(), key=operator.itemgetter(1), reverse=True)
# webResult = {}
#webResult[str(result_path + file_name)] = sorted_tags
result_url = urlparse(result_path).path
r.publish(
'chat',
json.dumps({
'web_result':
os.path.join(result_url, 'results',
file_name + '.mat'),
'socketid':
str(socketid)
}))
log_to_terminal('Thank you for using CloudCV', socketid)
# Single File
else:
""" Part 3/4
myPrint = CustomPrint(socketid)
old_stdout=sys.stdout
sys.stdout = myPrint
"""
tags = {}
print 'Running caffe classify on a single image: ' + single_file_name
try:
mat_file_path = decaf.calculate_decaf_image(
single_file_name, src_path, output_path, 3, socketid, tags,
modelname, modelnet)
except Exception as e:
print str(e)
""" Part 4/4
sys.stdout=old_stdout
"""
log_to_terminal("Results: " + str(tags), socketid)
# tags = sorted(tags.iteritems(), key=operator.itemgetter(1), reverse=True)
# web_result = {}
# web_result[str(result_path)] = tags
result_url = os.path.dirname(urlparse(result_path).path)
r.publish(
'chat',
json.dumps({
'web_result':
os.path.join(result_url, 'results',
single_file_name + '.mat'),
'socketid':
str(socketid)
}))
log_to_terminal('Thank you for using CloudCV', socketid)
except Exception as e:
log_to_terminal(str(traceback.format_exc()), socketid)
def trainModel(save_dir, socketid):
train_fast.modelUpdate(save_dir + '/')
log_to_terminal(
'Finished training your model with the new categories. Now, upload some test images to test this model. ',
socketid)
def featExtraction(src_path, socketid, result_url_prefix, feat_folder):
try:
#Entire Directory
if os.path.isdir(src_path):
for file_name in os.listdir(src_path):
image_path = os.path.join(src_path, file_name)
feat_path = os.path.join(feat_folder, file_name)
if os.path.isfile(image_path):
""" Trying to get the output of classify python script to send to user - Part 1/4
myPrint = CustomPrint(socketid)
old_stdout=sys.stdout
sys.stdout = myPrint
"""
if os.path.isfile(feat_path + '.npy'):
# Features already extracted for this file - it can actually be a different image but fix later
continue
print 'Extracting caffe features...'
caffe_feat_image(image_path, feat_path)
""" Part 2/2
sys.stdout=old_stdout
"""
log_to_terminal("Feature extraction done. ", socketid)
# tags = sorted(tags.iteritems(), key=operator.itemgetter(1),reverse=True)
webResult = {}
# webResult[str(os.path.join(result_url_prefix, file_name))] = [['Feature Extraction', 'Done']]
webResult["extracted"] = str(
os.path.join(result_url_prefix, file_name))
r.publish(
'chat',
json.dumps({
'web_result': json.dumps(webResult),
'socketid': str(socketid)
}))
# Single File
else:
""" Part 3/4
myPrint = CustomPrint(socketid)
old_stdout=sys.stdout
sys.stdout = myPrint
"""
print 'Extracting caffe features...'
# This is handled in vqa_views
feat_path = feat_folder
caffe_feat_image(src_path, feat_path)
""" Part 4/4
sys.stdout=old_stdout
"""
log_to_terminal("Feature extraction done. ", socketid)
# tags = sorted(tags.iteritems(), key=operator.itemgetter(1), reverse=True)
web_result = {}
# web_result[str(result_url_prefix)] = [['Feature Extraction', 'Done']]
web_result["extracted"] = str(result_url_prefix)
r.publish(
'chat',
json.dumps({
'web_result': json.dumps(web_result),
'socketid': str(socketid)
}))
except Exception as e:
log_to_terminal(str(traceback.format_exc()), socketid)
def saveFilesAndProcess(request, job_obj):
"""
TODO: Specify four types of location
1.) CloudCV General Dataset
2.) CloudCV me folder, me folder represents the user folder
3.) Dropbox folder
4.) Local System - Either save it as a test and delete, or permanently by specifying a name
"""
# If the server is different. The storage path needs to be changed to shared folders.
parsed_params = parseParameters(job_obj.params)
if 'server' in parsed_params and parsed_params['server'] == 'decaf_server':
job_obj.storage_path = '/srv/share/cloudcv/jobs/' + job_obj.userid + '/'
if not os.path.exists(job_obj.storage_path):
os.mkdir(job_obj.storage_path)
os.chmod(job_obj.storage_path, 0775)
job_obj.url = 'http://godel.ece.vt.edu/cloudcv/fileupload/media/pictures/decaf_server/' + job_obj.userid + \
'/' + job_obj.jobid
r.publish(
'chat',
json.dumps({
'error': str('Special Server Identified'),
'socketid': job_obj.socketid
}))
# Save files either through dropbox or client local system for use an throw scenario
if job_obj.dropbox_path is not None:
saveDropboxFiles.delay(job_obj.__dict__)
return 'Downloading content from dropbox. Execution will begin after downloading finishes'
else:
files_all = getFilesFromRequest(request, job_obj.count)
if len(files_all) == 0:
return 'Length of files = 0'
if len(files_all) > 50:
r.publish(
'chat',
json.dumps({
'error': str('Shutting down now.'),
'socketid': job_obj.socketid,
'end': 'yes'
}))
return 'Length of files higher than the limit of 50. Please use dropbox'
job_directory = os.path.join(job_obj.storage_path, str(job_obj.jobid))
log_to_terminal('Processing files', job_obj.socketid)
for single_file in files_all:
try:
# new_file_name = saveInPictureDatabase(single_file)
path = conf.PIC_DIR
size = (500, 500)
resizeImageAndTransfer(path, job_directory, size, single_file)
except Exception as e:
raise e
response = core_execute.execute(job_obj, job_directory, 'local')
return response
def featExtraction(src_path, socketid, result_url_prefix, feat_folder):
try:
#Entire Directory
if os.path.isdir(src_path):
for file_name in os.listdir(src_path):
image_path = os.path.join(src_path, file_name)
feat_path = os.path.join(feat_folder, file_name)
if os.path.isfile(image_path):
""" Trying to get the output of classify python script to send to user - Part 1/4
myPrint = CustomPrint(socketid)
old_stdout=sys.stdout
sys.stdout = myPrint
"""
if os.path.isfile(feat_path + '.npy'):
# Features already extracted for this file - it can actually be a different image but fix later
continue
print 'Extracting caffe features...'
caffe_feat_image(image_path, feat_path)
""" Part 2/2
sys.stdout=old_stdout
"""
log_to_terminal("Feature extraction done. ", socketid)
# tags = sorted(tags.iteritems(), key=operator.itemgetter(1),reverse=True)
webResult = {}
# webResult[str(os.path.join(result_url_prefix, file_name))] = [['Feature Extraction', 'Done']]
webResult["extracted"] = str(os.path.join(result_url_prefix, file_name))
r.publish('chat',
json.dumps({'web_result': json.dumps(webResult), 'socketid': str(socketid)}))
# Single File
else:
""" Part 3/4
myPrint = CustomPrint(socketid)
old_stdout=sys.stdout
sys.stdout = myPrint
"""
print 'Extracting caffe features...'
# This is handled in vqa_views
feat_path = feat_folder
caffe_feat_image(src_path, feat_path)
""" Part 4/4
sys.stdout=old_stdout
"""
log_to_terminal("Feature extraction done. ", socketid)
# tags = sorted(tags.iteritems(), key=operator.itemgetter(1), reverse=True)
web_result = {}
# web_result[str(result_url_prefix)] = [['Feature Extraction', 'Done']]
web_result["extracted"] = str(result_url_prefix)
r.publish('chat', json.dumps({'web_result': json.dumps(web_result), 'socketid': str(socketid)}))
except Exception as e:
log_to_terminal(str(traceback.format_exc()), socketid)
def classifyImages(src_path, socketid, result_path):
try:
#Entire Directory
if os.path.isdir(src_path):
for file_name in os.listdir(src_path):
image_path = os.path.join(src_path, file_name)
if os.path.isfile(image_path):
""" Trying to get the output of classify python script to send to user - Part 1/4
myPrint = CustomPrint(socketid)
old_stdout=sys.stdout
sys.stdout = myPrint
"""
print 'Running caffe classify...'
tags = caffe_classify_image(image_path)
""" Part 2/2
sys.stdout=old_stdout
"""
log_to_terminal("Results: " + str(tags), socketid)
# tags = sorted(tags.iteritems(), key=operator.itemgetter(1),reverse=True)
webResult = {}
webResult[str(os.path.join(result_path, file_name))] = tags
r.publish(
'chat',
json.dumps({
'web_result': json.dumps(webResult),
'socketid': str(socketid)
}))
log_to_terminal('Thank you for using CloudCV', socketid)
# Single File
else:
""" Part 3/4
myPrint = CustomPrint(socketid)
old_stdout=sys.stdout
sys.stdout = myPrint
"""
print 'Running caffe classify...'
tags = caffe_classify_image(src_path)
""" Part 4/4
sys.stdout=old_stdout
"""
log_to_terminal("Results: " + str(tags), socketid)
# tags = sorted(tags.iteritems(), key=operator.itemgetter(1), reverse=True)
web_result = {}
web_result[str(result_path)] = tags
r.publish(
'chat',
json.dumps({
'web_result': json.dumps(web_result),
'socketid': str(socketid)
}))
log_to_terminal('Thank you for using CloudCV', socketid)
except Exception as e:
log_to_terminal(str(traceback.format_exc()), socketid)
def downloadFiles(job_dict):
log_to_terminal('Dropbox Queue Handler: Downloading images',
job_dict['socketid'])
client = dropbox.client.DropboxClient(job_dict['dropbox_token'])
try:
folder_metadata = client.metadata(job_dict['dropbox_path'])
except Exception as e:
log_and_exit('Path: ' + job_dict['dropbox_path'] + ' not found',
job_dict['socketid'])
return
log(job_dict['dropbox_path'], downloadFiles.__name__)
log(job_dict['dropbox_token'], downloadFiles.__name__)
for content in folder_metadata['contents']:
if content['is_dir'] is False and 'image' in str(content['mime_type']):
file, file_metadata = client.get_file_and_metadata(content['path'])
rel_path = ntpath.dirname(str(content['path'])).lstrip('/')
log_to_terminal(
os.path.join(job_dict['storage_path'], rel_path,
ntpath.basename(content['path'])),
job_dict['socketid'])
if not os.path.exists(
os.path.join(job_dict['storage_path'], rel_path)):
os.makedirs(os.path.join(job_dict['storage_path'], rel_path))
os.chmod(os.path.join(job_dict['storage_path'], rel_path),
0775)
img = Image.open(StringIO(file.read()))
img.save(
os.path.join(job_dict['storage_path'], rel_path,
ntpath.basename(content['path'])),
file_metadata['mime_type'].split('/')[1])
try:
job_obj = Job()
for k, v in job_dict.iteritems():
setattr(job_obj, k, v)
log(
str(job_dict['storage_path'] +
job_dict['dropbox_path'].strip('/')), 'directory')
if job_obj.count is None or int(job_obj.count) <= 0:
count = 0
for name in os.listdir(
os.path.join(job_dict['storage_path'],
job_dict['dropbox_path'].strip('/'))):
log(str(name), 'name')
if os.path.isfile(
os.path.join(job_dict['storage_path'],
job_dict['dropbox_path'].strip('/'),
name)):
count += 1
log(str(name), 'name')
job_obj.count = count
# log_to_terminal('JobID: ' + str(job_obj.jobid), job_obj.socketid)
# log_to_terminal('Count: ' + str(job_obj.count), job_obj.socketid)
core_execute.execute(
job_obj,
os.path.join(job_dict['storage_path'],
job_dict['dropbox_path'].strip('/')), 'dropbox')
except Exception as e:
print str(traceback.format_exc())
log_and_exit(str(traceback.format_exc()), job_dict['socketid'])
