def execute(job_obj, image_path=None, source_type=None):
try:
directory = os.path.join(job_obj.storage_path, str(job_obj.jobid))
if not os.path.exists(directory):
os.makedirs(directory)
os.chmod(directory, 0776)
result_path = directory
# Run an executable as defined by the user
if job_obj.url is None:
job_obj.url = os.path.join(conf.PIC_URL, job_obj.userid, job_obj.jobid)
log_to_terminal('Files Stored. Waiting for one of the worker nodes to handle this request', job_obj.socketid)
parsed_dict = {'jobid': job_obj.jobid, 'userid': job_obj.userid,
'image_path': str(image_path), 'result_path': str(result_path),
'dir': str(directory), 'url': job_obj.url,
'source_type': source_type, 'exec': job_obj.executable,
'count': job_obj.count, 'token': job_obj.token,
'socketid': job_obj.socketid, 'params': job_obj.params,
'dropbox_token': job_obj.dropbox_token}
r.publish('chat', json.dumps({'jobinfo': parsed_dict, 'socketid': str(job_obj.socketid)}))
run.delay(parsed_dict)
except:
log_and_exit(str(traceback.format_exc()), job_obj.socketid)
return str('Error pushing to the job queue')
return str(parsed_dict)
def downloadFiles(job_dict):
log_to_terminal('Dropbox Queue Handler: Downloading images', job_dict['socketid'])
client = dropbox.client.DropboxClient(job_dict['dropbox_token'])
try:
folder_metadata = client.metadata(job_dict['dropbox_path'])
except Exception as e:
log_and_exit('Path: ' + job_dict['dropbox_path'] + ' not found', job_dict['socketid'])
return
log(job_dict['dropbox_path'], downloadFiles.__name__)
log(job_dict['dropbox_token'], downloadFiles.__name__)
for content in folder_metadata['contents']:
if content['is_dir'] is False and 'image' in str(content['mime_type']):
file, file_metadata = client.get_file_and_metadata(content['path'])
rel_path = ntpath.dirname(str(content['path'])).lstrip('/')
log_to_terminal(os.path.join(job_dict['storage_path'] ,rel_path , ntpath.basename(content['path'])),
job_dict['socketid'])
if not os.path.exists(os.path.join(job_dict['storage_path'], rel_path)):
os.makedirs(os.path.join(job_dict['storage_path'], rel_path))
os.chmod(os.path.join(job_dict['storage_path'] , rel_path), 0775)
img = Image.open(StringIO(file.read()))
img.save(os.path.join(job_dict['storage_path'], rel_path,
ntpath.basename(content['path'])),
file_metadata['mime_type'].split('/')[1])
try:
job_obj = Job()
for k,v in job_dict.iteritems():
setattr(job_obj, k, v)
log(str(job_dict['storage_path'] + job_dict['dropbox_path'].strip('/')), 'directory')
if job_obj.count is None or int(job_obj.count) <= 0:
count = 0
for name in os.listdir(os.path.join(job_dict['storage_path'], job_dict['dropbox_path'].strip('/'))):
log(str(name), 'name')
if os.path.isfile(os.path.join(job_dict['storage_path'], job_dict['dropbox_path'].strip('/'), name)):
count +=1
log(str(name), 'name')
job_obj.count = count
# log_to_terminal('JobID: ' + str(job_obj.jobid), job_obj.socketid)
# log_to_terminal('Count: ' + str(job_obj.count), job_obj.socketid)
core_execute.execute(job_obj, os.path.join(job_dict['storage_path'], job_dict['dropbox_path'].strip('/')), 'dropbox')
except Exception as e:
print str(traceback.format_exc())
log_and_exit(str(traceback.format_exc()), job_dict['socketid'])
def downloadFiles(job_dict):
log_to_terminal('Dropbox Queue Handler: Downloading images',
job_dict['socketid'])
client = dropbox.client.DropboxClient(job_dict['dropbox_token'])
try:
folder_metadata = client.metadata(job_dict['dropbox_path'])
except Exception as e:
log_and_exit('Path: ' + job_dict['dropbox_path'] + ' not found',
job_dict['socketid'])
return
log(job_dict['dropbox_path'], downloadFiles.__name__)
log(job_dict['dropbox_token'], downloadFiles.__name__)
for content in folder_metadata['contents']:
if content['is_dir'] is False and 'image' in str(content['mime_type']):
file, file_metadata = client.get_file_and_metadata(content['path'])
rel_path = ntpath.dirname(str(content['path'])).lstrip('/')
log_to_terminal(
os.path.join(job_dict['storage_path'], rel_path,
ntpath.basename(content['path'])),
job_dict['socketid'])
if not os.path.exists(
os.path.join(job_dict['storage_path'], rel_path)):
os.makedirs(os.path.join(job_dict['storage_path'], rel_path))
os.chmod(os.path.join(job_dict['storage_path'], rel_path),
0775)
img = Image.open(StringIO(file.read()))
img.save(
os.path.join(job_dict['storage_path'], rel_path,
ntpath.basename(content['path'])),
file_metadata['mime_type'].split('/')[1])
try:
job_obj = Job()
for k, v in job_dict.iteritems():
setattr(job_obj, k, v)
log(
str(job_dict['storage_path'] +
job_dict['dropbox_path'].strip('/')), 'directory')
if job_obj.count is None or int(job_obj.count) <= 0:
count = 0
for name in os.listdir(
os.path.join(job_dict['storage_path'],
job_dict['dropbox_path'].strip('/'))):
log(str(name), 'name')
if os.path.isfile(
os.path.join(job_dict['storage_path'],
job_dict['dropbox_path'].strip('/'),
name)):
count += 1
log(str(name), 'name')
job_obj.count = count
# log_to_terminal('JobID: ' + str(job_obj.jobid), job_obj.socketid)
# log_to_terminal('Count: ' + str(job_obj.count), job_obj.socketid)
core_execute.execute(
job_obj,
os.path.join(job_dict['storage_path'],
job_dict['dropbox_path'].strip('/')), 'dropbox')
except Exception as e:
print str(traceback.format_exc())
log_and_exit(str(traceback.format_exc()), job_dict['socketid'])
def run(parsed_dict):
socketid = str(parsed_dict['socketid'])
try:
if 'dropbox_token' not in parsed_dict or parsed_dict['dropbox_token']=='None':
parsed_dict['dropbox_token'] = None
global jobid
jobid = str(parsed_dict['jobid'])
userid = str(parsed_dict['userid'])
image_path = parsed_dict['image_path']
result_path = str(parsed_dict['result_path'])
list = None
flag = None
server = None
dict_of_param = createList(image_path, parsed_dict)
if 'dict' in dict_of_param:
list = dict_of_param['dict']
if 'flag' in dict_of_param:
flag = dict_of_param['flag']
if 'server' in dict_of_param:
server = dict_of_param['server']
log(list, "__main__")
token = parsed_dict['token']
result_url = os.path.join(parsed_dict['url'], 'results')
result_path = os.path.join(result_path, 'results')
source_type = parsed_dict['source_type']
db_token = parsed_dict['dropbox_token']
if parsed_dict['exec'] == 'ImageStitch':
print "Image Stitching running"
output = run_executable(list, True, socketid)
sendsMessageToRedis(userid, jobid, source_type, socketid, output, result_path, result_url,
dropbox_token=db_token)
r.publish('chat', json.dumps({'done': str('Image Stitching done'), 'socketid': str(socketid), 'token': token, 'jobid': jobid}))
elif(parsed_dict['exec'] == 'VOCRelease5'):
# output = run_executable(list)
output = run_matlab_code(mlab_obj, '/var/www/html/cloudcv/voc-dpm-matlab-bridge/pascal_object_detection.m', list, parsed_dict['socketid'])
sendsMessageToRedis(userid, jobid, source_type, socketid, output, result_path, result_url,
dropbox_token=db_token)
r.publish('chat', json.dumps({'message': str('Bounding Boxes Generated'), 'socketid': str(socketid), 'token': token, 'jobid': jobid}))
elif(parsed_dict['exec'] == 'classify'):
run_classification(parsed_dict['userid'], parsed_dict['jobid'], parsed_dict['image_path'],
parsed_dict['socketid'], parsed_dict['token'], parsed_dict['source_type'],result_path,
db_token=db_token)
elif(parsed_dict['exec'] == 'features'):
print result_url
tags = {}
matlabfilepath = decaf_cal_feature.calculate_decaf(parsed_dict['image_path'], result_path,3,socketid, tags)
sendsMessageToRedis(userid, jobid, source_type, socketid, '', result_path, result_url,
dropbox_token=db_token)
r.publish('chat', json.dumps({'done': str('Features Generated'),
'socketid': str(socketid), 'token': token, 'jobid': jobid}))
except Exception as e:
log_and_exit(str(traceback.format_exc()), socketid)
def run(parsed_dict):
socketid = str(parsed_dict['socketid'])
try:
if 'dropbox_token' not in parsed_dict or parsed_dict[
'dropbox_token'] == 'None':
parsed_dict['dropbox_token'] = None
global jobid
jobid = str(parsed_dict['jobid'])
userid = str(parsed_dict['userid'])
image_path = parsed_dict['image_path']
result_path = str(parsed_dict['result_path'])
list = None
flag = None
server = None
dict_of_param = createList(image_path, parsed_dict)
if 'dict' in dict_of_param:
list = dict_of_param['dict']
if 'flag' in dict_of_param:
flag = dict_of_param['flag']
if 'server' in dict_of_param:
server = dict_of_param['server']
log(list, "__main__")
token = parsed_dict['token']
result_url = os.path.join(parsed_dict['url'], 'results')
result_path = os.path.join(result_path, 'results')
source_type = parsed_dict['source_type']
db_token = parsed_dict['dropbox_token']
if parsed_dict['exec'] == 'ImageStitch':
print "Image Stitching running"
output = run_executable(list, True, socketid)
sendsMessageToRedis(userid,
jobid,
source_type,
socketid,
output,
result_path,
result_url,
dropbox_token=db_token)
r.publish(
'chat',
json.dumps({
'done': str('Image Stitching done'),
'socketid': str(socketid),
'token': token,
'jobid': jobid
}))
elif (parsed_dict['exec'] == 'VOCRelease5'):
# output = run_executable(list)
output = run_matlab_code(
mlab_obj,
'/var/www/html/cloudcv/voc-dpm-matlab-bridge/pascal_object_detection.m',
list, parsed_dict['socketid'])
sendsMessageToRedis(userid,
jobid,
source_type,
socketid,
output,
result_path,
result_url,
dropbox_token=db_token)
r.publish(
'chat',
json.dumps({
'message': str('Bounding Boxes Generated'),
'socketid': str(socketid),
'token': token,
'jobid': jobid
}))
elif (parsed_dict['exec'] == 'classify'):
run_classification(parsed_dict['userid'],
parsed_dict['jobid'],
parsed_dict['image_path'],
parsed_dict['socketid'],
parsed_dict['token'],
parsed_dict['source_type'],
result_path,
db_token=db_token)
elif (parsed_dict['exec'] == 'features'):
print result_url
tags = {}
matlabfilepath = decaf_cal_feature.calculate_decaf(
parsed_dict['image_path'], result_path, 3, socketid, tags)
sendsMessageToRedis(userid,
jobid,
source_type,
socketid,
'',
result_path,
result_url,
dropbox_token=db_token)
r.publish(
'chat',
json.dumps({
'done': str('Features Generated'),
'socketid': str(socketid),
'token': token,
'jobid': jobid
}))
except Exception as e:
log_and_exit(str(traceback.format_exc()), socketid)