def downloadFiles(job_dict):
log_to_terminal('Dropbox Queue Handler: Downloading images', job_dict['socketid'])
client = dropbox.client.DropboxClient(job_dict['dropbox_token'])
try:
folder_metadata = client.metadata(job_dict['dropbox_path'])
except Exception as e:
log_and_exit('Path: ' + job_dict['dropbox_path'] + ' not found', job_dict['socketid'])
return
log(job_dict['dropbox_path'], downloadFiles.__name__)
log(job_dict['dropbox_token'], downloadFiles.__name__)
for content in folder_metadata['contents']:
if content['is_dir'] is False and 'image' in str(content['mime_type']):
file, file_metadata = client.get_file_and_metadata(content['path'])
rel_path = ntpath.dirname(str(content['path'])).lstrip('/')
log_to_terminal(os.path.join(job_dict['storage_path'] ,rel_path , ntpath.basename(content['path'])),
job_dict['socketid'])
if not os.path.exists(os.path.join(job_dict['storage_path'], rel_path)):
os.makedirs(os.path.join(job_dict['storage_path'], rel_path))
os.chmod(os.path.join(job_dict['storage_path'] , rel_path), 0775)
img = Image.open(StringIO(file.read()))
img.save(os.path.join(job_dict['storage_path'], rel_path,
ntpath.basename(content['path'])),
file_metadata['mime_type'].split('/')[1])
try:
job_obj = Job()
for k,v in job_dict.iteritems():
setattr(job_obj, k, v)
log(str(job_dict['storage_path'] + job_dict['dropbox_path'].strip('/')), 'directory')
if job_obj.count is None or int(job_obj.count) <= 0:
count = 0
for name in os.listdir(os.path.join(job_dict['storage_path'], job_dict['dropbox_path'].strip('/'))):
log(str(name), 'name')
if os.path.isfile(os.path.join(job_dict['storage_path'], job_dict['dropbox_path'].strip('/'), name)):
count +=1
log(str(name), 'name')
job_obj.count = count
# log_to_terminal('JobID: ' + str(job_obj.jobid), job_obj.socketid)
# log_to_terminal('Count: ' + str(job_obj.count), job_obj.socketid)
core_execute.execute(job_obj, os.path.join(job_dict['storage_path'], job_dict['dropbox_path'].strip('/')), 'dropbox')
except Exception as e:
print str(traceback.format_exc())
log_and_exit(str(traceback.format_exc()), job_dict['socketid'])
def saveFilesAndProcess(request, job_obj):
"""
TODO: Specify four types of location
1.) CloudCV General Dataset
2.) CloudCV me folder, me folder represents the user folder
3.) Dropbox folder
4.) Local System - Either save it as a test and delete, or permanently by specifying a name
"""
# If the server is different. The storage path needs to be changed to shared folders.
parsed_params = parseParameters(job_obj.params)
if 'server' in parsed_params and parsed_params['server'] == 'decaf_server':
job_obj.storage_path = '/srv/share/cloudcv/jobs/' + job_obj.userid + '/'
if not os.path.exists(job_obj.storage_path):
os.mkdir(job_obj.storage_path)
os.chmod(job_obj.storage_path, 0775)
job_obj.url = 'http://godel.ece.vt.edu/cloudcv/fileupload/media/pictures/decaf_server/'+ job_obj.userid + \
'/' + job_obj.jobid
r.publish('chat', json.dumps({'error': str('Special Server Identified'), 'socketid': job_obj.socketid}))
# Save files either through dropbox or client local system for use an throw scenario
if job_obj.dropbox_path is not None:
result = saveDropboxFiles.delay(job_obj.__dict__)
return 'Downloading content from dropbox. Execution will begin after downloading finishes'
else:
files_all = getFilesFromRequest(request, job_obj.count)
if len(files_all) == 0:
return 'Length of files = 0'
if len(files_all) > 50:
r.publish('chat', json.dumps({'error': str('Shutting down now.'),
'socketid':job_obj.socketid, 'end': 'yes'}))
return 'Length of files higher than the limit of 50. Please use dropbox'
job_directory = os.path.join(job_obj.storage_path, str(job_obj.jobid))
log_to_terminal('Processing files', job_obj.socketid)
for single_file in files_all:
try:
# new_file_name = saveInPictureDatabase(single_file)
path = conf.PIC_DIR
size = (500, 500)
resizeImageAndTransfer(path, job_directory, size, single_file)
except Exception as e:
print str(traceback.format_exc())
raise e
response = core_execute.execute(job_obj, job_directory, 'local')
return response
def saveFilesAndProcess(request, job_obj):
"""
TODO: Specify four types of location
1.) CloudCV General Dataset
2.) CloudCV me folder, me folder represents the user folder
3.) Dropbox folder
4.) Local System - Either save it as a test and delete, or permanently by specifying a name
"""
# If the server is different. The storage path needs to be changed to shared folders.
parsed_params = parseParameters(job_obj.params)
if 'server' in parsed_params and parsed_params['server'] == 'decaf_server':
job_obj.storage_path = '/srv/share/cloudcv/jobs/' + job_obj.userid + '/'
if not os.path.exists(job_obj.storage_path):
os.mkdir(job_obj.storage_path)
os.chmod(job_obj.storage_path, 0775)
job_obj.url = 'http://godel.ece.vt.edu/cloudcv/fileupload/media/pictures/decaf_server/' + job_obj.userid + \
'/' + job_obj.jobid
r.publish(
'chat',
json.dumps({
'error': str('Special Server Identified'),
'socketid': job_obj.socketid
}))
# Save files either through dropbox or client local system for use an throw scenario
if job_obj.dropbox_path is not None:
saveDropboxFiles.delay(job_obj.__dict__)
return 'Downloading content from dropbox. Execution will begin after downloading finishes'
else:
files_all = getFilesFromRequest(request, job_obj.count)
if len(files_all) == 0:
return 'Length of files = 0'
if len(files_all) > 50:
r.publish(
'chat',
json.dumps({
'error': str('Shutting down now.'),
'socketid': job_obj.socketid,
'end': 'yes'
}))
return 'Length of files higher than the limit of 50. Please use dropbox'
job_directory = os.path.join(job_obj.storage_path, str(job_obj.jobid))
log_to_terminal('Processing files', job_obj.socketid)
for single_file in files_all:
try:
# new_file_name = saveInPictureDatabase(single_file)
path = conf.PIC_DIR
size = (500, 500)
resizeImageAndTransfer(path, job_directory, size, single_file)
except Exception as e:
raise e
response = core_execute.execute(job_obj, job_directory, 'local')
return response
def downloadFiles(job_dict):
log_to_terminal('Dropbox Queue Handler: Downloading images',
job_dict['socketid'])
client = dropbox.client.DropboxClient(job_dict['dropbox_token'])
try:
folder_metadata = client.metadata(job_dict['dropbox_path'])
except Exception as e:
log_and_exit('Path: ' + job_dict['dropbox_path'] + ' not found',
job_dict['socketid'])
return
log(job_dict['dropbox_path'], downloadFiles.__name__)
log(job_dict['dropbox_token'], downloadFiles.__name__)
for content in folder_metadata['contents']:
if content['is_dir'] is False and 'image' in str(content['mime_type']):
file, file_metadata = client.get_file_and_metadata(content['path'])
rel_path = ntpath.dirname(str(content['path'])).lstrip('/')
log_to_terminal(
os.path.join(job_dict['storage_path'], rel_path,
ntpath.basename(content['path'])),
job_dict['socketid'])
if not os.path.exists(
os.path.join(job_dict['storage_path'], rel_path)):
os.makedirs(os.path.join(job_dict['storage_path'], rel_path))
os.chmod(os.path.join(job_dict['storage_path'], rel_path),
0775)
img = Image.open(StringIO(file.read()))
img.save(
os.path.join(job_dict['storage_path'], rel_path,
ntpath.basename(content['path'])),
file_metadata['mime_type'].split('/')[1])
try:
job_obj = Job()
for k, v in job_dict.iteritems():
setattr(job_obj, k, v)
log(
str(job_dict['storage_path'] +
job_dict['dropbox_path'].strip('/')), 'directory')
if job_obj.count is None or int(job_obj.count) <= 0:
count = 0
for name in os.listdir(
os.path.join(job_dict['storage_path'],
job_dict['dropbox_path'].strip('/'))):
log(str(name), 'name')
if os.path.isfile(
os.path.join(job_dict['storage_path'],
job_dict['dropbox_path'].strip('/'),
name)):
count += 1
log(str(name), 'name')
job_obj.count = count
# log_to_terminal('JobID: ' + str(job_obj.jobid), job_obj.socketid)
# log_to_terminal('Count: ' + str(job_obj.count), job_obj.socketid)
core_execute.execute(
job_obj,
os.path.join(job_dict['storage_path'],
job_dict['dropbox_path'].strip('/')), 'dropbox')
except Exception as e:
print str(traceback.format_exc())
log_and_exit(str(traceback.format_exc()), job_dict['socketid'])
