def downloadFiles(job_dict): log_to_terminal('Dropbox Queue Handler: Downloading images', job_dict['socketid']) client = dropbox.client.DropboxClient(job_dict['dropbox_token']) try: folder_metadata = client.metadata(job_dict['dropbox_path']) except Exception as e: log_and_exit('Path: ' + job_dict['dropbox_path'] + ' not found', job_dict['socketid']) return log(job_dict['dropbox_path'], downloadFiles.__name__) log(job_dict['dropbox_token'], downloadFiles.__name__) for content in folder_metadata['contents']: if content['is_dir'] is False and 'image' in str(content['mime_type']): file, file_metadata = client.get_file_and_metadata(content['path']) rel_path = ntpath.dirname(str(content['path'])).lstrip('/') log_to_terminal(os.path.join(job_dict['storage_path'] ,rel_path , ntpath.basename(content['path'])), job_dict['socketid']) if not os.path.exists(os.path.join(job_dict['storage_path'], rel_path)): os.makedirs(os.path.join(job_dict['storage_path'], rel_path)) os.chmod(os.path.join(job_dict['storage_path'] , rel_path), 0775) img = Image.open(StringIO(file.read())) img.save(os.path.join(job_dict['storage_path'], rel_path, ntpath.basename(content['path'])), file_metadata['mime_type'].split('/')[1]) try: job_obj = Job() for k,v in job_dict.iteritems(): setattr(job_obj, k, v) log(str(job_dict['storage_path'] + job_dict['dropbox_path'].strip('/')), 'directory') if job_obj.count is None or int(job_obj.count) <= 0: count = 0 for name in os.listdir(os.path.join(job_dict['storage_path'], job_dict['dropbox_path'].strip('/'))): log(str(name), 'name') if os.path.isfile(os.path.join(job_dict['storage_path'], job_dict['dropbox_path'].strip('/'), name)): count +=1 log(str(name), 'name') job_obj.count = count # log_to_terminal('JobID: ' + str(job_obj.jobid), job_obj.socketid) # log_to_terminal('Count: ' + str(job_obj.count), job_obj.socketid) core_execute.execute(job_obj, os.path.join(job_dict['storage_path'], job_dict['dropbox_path'].strip('/')), 'dropbox') except Exception as e: print str(traceback.format_exc()) log_and_exit(str(traceback.format_exc()), job_dict['socketid'])
def saveFilesAndProcess(request, job_obj): """ TODO: Specify four types of location 1.) CloudCV General Dataset 2.) CloudCV me folder, me folder represents the user folder 3.) Dropbox folder 4.) Local System - Either save it as a test and delete, or permanently by specifying a name """ # If the server is different. The storage path needs to be changed to shared folders. parsed_params = parseParameters(job_obj.params) if 'server' in parsed_params and parsed_params['server'] == 'decaf_server': job_obj.storage_path = '/srv/share/cloudcv/jobs/' + job_obj.userid + '/' if not os.path.exists(job_obj.storage_path): os.mkdir(job_obj.storage_path) os.chmod(job_obj.storage_path, 0775) job_obj.url = 'http://godel.ece.vt.edu/cloudcv/fileupload/media/pictures/decaf_server/'+ job_obj.userid + \ '/' + job_obj.jobid r.publish('chat', json.dumps({'error': str('Special Server Identified'), 'socketid': job_obj.socketid})) # Save files either through dropbox or client local system for use an throw scenario if job_obj.dropbox_path is not None: result = saveDropboxFiles.delay(job_obj.__dict__) return 'Downloading content from dropbox. Execution will begin after downloading finishes' else: files_all = getFilesFromRequest(request, job_obj.count) if len(files_all) == 0: return 'Length of files = 0' if len(files_all) > 50: r.publish('chat', json.dumps({'error': str('Shutting down now.'), 'socketid':job_obj.socketid, 'end': 'yes'})) return 'Length of files higher than the limit of 50. Please use dropbox' job_directory = os.path.join(job_obj.storage_path, str(job_obj.jobid)) log_to_terminal('Processing files', job_obj.socketid) for single_file in files_all: try: # new_file_name = saveInPictureDatabase(single_file) path = conf.PIC_DIR size = (500, 500) resizeImageAndTransfer(path, job_directory, size, single_file) except Exception as e: print str(traceback.format_exc()) raise e response = core_execute.execute(job_obj, job_directory, 'local') return response
def saveFilesAndProcess(request, job_obj): """ TODO: Specify four types of location 1.) CloudCV General Dataset 2.) CloudCV me folder, me folder represents the user folder 3.) Dropbox folder 4.) Local System - Either save it as a test and delete, or permanently by specifying a name """ # If the server is different. The storage path needs to be changed to shared folders. parsed_params = parseParameters(job_obj.params) if 'server' in parsed_params and parsed_params['server'] == 'decaf_server': job_obj.storage_path = '/srv/share/cloudcv/jobs/' + job_obj.userid + '/' if not os.path.exists(job_obj.storage_path): os.mkdir(job_obj.storage_path) os.chmod(job_obj.storage_path, 0775) job_obj.url = 'http://godel.ece.vt.edu/cloudcv/fileupload/media/pictures/decaf_server/' + job_obj.userid + \ '/' + job_obj.jobid r.publish( 'chat', json.dumps({ 'error': str('Special Server Identified'), 'socketid': job_obj.socketid })) # Save files either through dropbox or client local system for use an throw scenario if job_obj.dropbox_path is not None: saveDropboxFiles.delay(job_obj.__dict__) return 'Downloading content from dropbox. Execution will begin after downloading finishes' else: files_all = getFilesFromRequest(request, job_obj.count) if len(files_all) == 0: return 'Length of files = 0' if len(files_all) > 50: r.publish( 'chat', json.dumps({ 'error': str('Shutting down now.'), 'socketid': job_obj.socketid, 'end': 'yes' })) return 'Length of files higher than the limit of 50. Please use dropbox' job_directory = os.path.join(job_obj.storage_path, str(job_obj.jobid)) log_to_terminal('Processing files', job_obj.socketid) for single_file in files_all: try: # new_file_name = saveInPictureDatabase(single_file) path = conf.PIC_DIR size = (500, 500) resizeImageAndTransfer(path, job_directory, size, single_file) except Exception as e: raise e response = core_execute.execute(job_obj, job_directory, 'local') return response
def downloadFiles(job_dict): log_to_terminal('Dropbox Queue Handler: Downloading images', job_dict['socketid']) client = dropbox.client.DropboxClient(job_dict['dropbox_token']) try: folder_metadata = client.metadata(job_dict['dropbox_path']) except Exception as e: log_and_exit('Path: ' + job_dict['dropbox_path'] + ' not found', job_dict['socketid']) return log(job_dict['dropbox_path'], downloadFiles.__name__) log(job_dict['dropbox_token'], downloadFiles.__name__) for content in folder_metadata['contents']: if content['is_dir'] is False and 'image' in str(content['mime_type']): file, file_metadata = client.get_file_and_metadata(content['path']) rel_path = ntpath.dirname(str(content['path'])).lstrip('/') log_to_terminal( os.path.join(job_dict['storage_path'], rel_path, ntpath.basename(content['path'])), job_dict['socketid']) if not os.path.exists( os.path.join(job_dict['storage_path'], rel_path)): os.makedirs(os.path.join(job_dict['storage_path'], rel_path)) os.chmod(os.path.join(job_dict['storage_path'], rel_path), 0775) img = Image.open(StringIO(file.read())) img.save( os.path.join(job_dict['storage_path'], rel_path, ntpath.basename(content['path'])), file_metadata['mime_type'].split('/')[1]) try: job_obj = Job() for k, v in job_dict.iteritems(): setattr(job_obj, k, v) log( str(job_dict['storage_path'] + job_dict['dropbox_path'].strip('/')), 'directory') if job_obj.count is None or int(job_obj.count) <= 0: count = 0 for name in os.listdir( os.path.join(job_dict['storage_path'], job_dict['dropbox_path'].strip('/'))): log(str(name), 'name') if os.path.isfile( os.path.join(job_dict['storage_path'], job_dict['dropbox_path'].strip('/'), name)): count += 1 log(str(name), 'name') job_obj.count = count # log_to_terminal('JobID: ' + str(job_obj.jobid), job_obj.socketid) # log_to_terminal('Count: ' + str(job_obj.count), job_obj.socketid) core_execute.execute( job_obj, os.path.join(job_dict['storage_path'], job_dict['dropbox_path'].strip('/')), 'dropbox') except Exception as e: print str(traceback.format_exc()) log_and_exit(str(traceback.format_exc()), job_dict['socketid'])