Exemple #1
0
def execute(job_obj, image_path=None, source_type=None):
    try:
        directory = os.path.join(job_obj.storage_path, str(job_obj.jobid))

        if not os.path.exists(directory):
            os.makedirs(directory)
            os.chmod(directory, 0776)

        result_path = directory

        # Run an executable as defined by the user
        if job_obj.url is None:
            job_obj.url = os.path.join(conf.PIC_URL, job_obj.userid, job_obj.jobid)

        log_to_terminal('Files Stored. Waiting for one of the worker nodes to handle this request', job_obj.socketid)
        parsed_dict = {'jobid': job_obj.jobid, 'userid': job_obj.userid,
                       'image_path': str(image_path), 'result_path': str(result_path),
                       'dir': str(directory), 'url': job_obj.url,
                       'source_type': source_type, 'exec': job_obj.executable,
                       'count': job_obj.count, 'token': job_obj.token,
                       'socketid': job_obj.socketid, 'params': job_obj.params,
                       'dropbox_token': job_obj.dropbox_token}

        r.publish('chat', json.dumps({'jobinfo': parsed_dict, 'socketid': str(job_obj.socketid)}))
        run.delay(parsed_dict)

    except:
        log_and_exit(str(traceback.format_exc()), job_obj.socketid)
        return str('Error pushing to the job queue')

    return str(parsed_dict)
def downloadFiles(job_dict):

    log_to_terminal('Dropbox Queue Handler: Downloading images', job_dict['socketid'])

    client = dropbox.client.DropboxClient(job_dict['dropbox_token'])
    try:
        folder_metadata = client.metadata(job_dict['dropbox_path'])
    except Exception as e:
        log_and_exit('Path: ' + job_dict['dropbox_path'] + ' not found', job_dict['socketid'])
        return

    log(job_dict['dropbox_path'], downloadFiles.__name__)
    log(job_dict['dropbox_token'], downloadFiles.__name__)

    for content in folder_metadata['contents']:
        if content['is_dir'] is False and 'image' in str(content['mime_type']):
            file, file_metadata = client.get_file_and_metadata(content['path'])

            rel_path = ntpath.dirname(str(content['path'])).lstrip('/')
            log_to_terminal(os.path.join(job_dict['storage_path'] ,rel_path , ntpath.basename(content['path'])),
                            job_dict['socketid'])

            if not os.path.exists(os.path.join(job_dict['storage_path'], rel_path)):
                os.makedirs(os.path.join(job_dict['storage_path'],  rel_path))
                os.chmod(os.path.join(job_dict['storage_path'] , rel_path), 0775)

            img = Image.open(StringIO(file.read()))
            img.save(os.path.join(job_dict['storage_path'], rel_path,
                                  ntpath.basename(content['path'])),
                                  file_metadata['mime_type'].split('/')[1])


    try:
        job_obj = Job()
        for k,v in job_dict.iteritems():
            setattr(job_obj, k, v)

        log(str(job_dict['storage_path'] + job_dict['dropbox_path'].strip('/')), 'directory')

        if job_obj.count is None or int(job_obj.count) <= 0:
            count = 0
            for name in os.listdir(os.path.join(job_dict['storage_path'], job_dict['dropbox_path'].strip('/'))):
                log(str(name), 'name')
                if os.path.isfile(os.path.join(job_dict['storage_path'], job_dict['dropbox_path'].strip('/'),  name)):
                    count +=1
                    log(str(name), 'name')
            job_obj.count = count

        # log_to_terminal('JobID: ' + str(job_obj.jobid), job_obj.socketid)
        # log_to_terminal('Count: ' + str(job_obj.count), job_obj.socketid)

        core_execute.execute(job_obj, os.path.join(job_dict['storage_path'], job_dict['dropbox_path'].strip('/')), 'dropbox')
    except Exception as e:
        print str(traceback.format_exc())
        log_and_exit(str(traceback.format_exc()), job_dict['socketid'])
Exemple #3
0
def downloadFiles(job_dict):

    log_to_terminal('Dropbox Queue Handler: Downloading images',
                    job_dict['socketid'])

    client = dropbox.client.DropboxClient(job_dict['dropbox_token'])
    try:
        folder_metadata = client.metadata(job_dict['dropbox_path'])
    except Exception as e:
        log_and_exit('Path: ' + job_dict['dropbox_path'] + ' not found',
                     job_dict['socketid'])
        return

    log(job_dict['dropbox_path'], downloadFiles.__name__)
    log(job_dict['dropbox_token'], downloadFiles.__name__)

    for content in folder_metadata['contents']:
        if content['is_dir'] is False and 'image' in str(content['mime_type']):
            file, file_metadata = client.get_file_and_metadata(content['path'])

            rel_path = ntpath.dirname(str(content['path'])).lstrip('/')
            log_to_terminal(
                os.path.join(job_dict['storage_path'], rel_path,
                             ntpath.basename(content['path'])),
                job_dict['socketid'])

            if not os.path.exists(
                    os.path.join(job_dict['storage_path'], rel_path)):
                os.makedirs(os.path.join(job_dict['storage_path'], rel_path))
                os.chmod(os.path.join(job_dict['storage_path'], rel_path),
                         0775)

            img = Image.open(StringIO(file.read()))
            img.save(
                os.path.join(job_dict['storage_path'], rel_path,
                             ntpath.basename(content['path'])),
                file_metadata['mime_type'].split('/')[1])

    try:
        job_obj = Job()
        for k, v in job_dict.iteritems():
            setattr(job_obj, k, v)

        log(
            str(job_dict['storage_path'] +
                job_dict['dropbox_path'].strip('/')), 'directory')

        if job_obj.count is None or int(job_obj.count) <= 0:
            count = 0
            for name in os.listdir(
                    os.path.join(job_dict['storage_path'],
                                 job_dict['dropbox_path'].strip('/'))):
                log(str(name), 'name')
                if os.path.isfile(
                        os.path.join(job_dict['storage_path'],
                                     job_dict['dropbox_path'].strip('/'),
                                     name)):
                    count += 1
                    log(str(name), 'name')
            job_obj.count = count

        # log_to_terminal('JobID: ' + str(job_obj.jobid), job_obj.socketid)
        # log_to_terminal('Count: ' + str(job_obj.count), job_obj.socketid)

        core_execute.execute(
            job_obj,
            os.path.join(job_dict['storage_path'],
                         job_dict['dropbox_path'].strip('/')), 'dropbox')
    except Exception as e:
        print str(traceback.format_exc())
        log_and_exit(str(traceback.format_exc()), job_dict['socketid'])
Exemple #4
0
def run(parsed_dict):
    socketid = str(parsed_dict['socketid'])
    try:
        if 'dropbox_token' not in parsed_dict or parsed_dict['dropbox_token']=='None':
            parsed_dict['dropbox_token'] = None

        global jobid

        jobid = str(parsed_dict['jobid'])
        userid = str(parsed_dict['userid'])

        image_path = parsed_dict['image_path']
        result_path = str(parsed_dict['result_path'])

        list = None
        flag = None
        server = None

        dict_of_param = createList(image_path, parsed_dict)
        if 'dict' in dict_of_param:
            list = dict_of_param['dict']
        if 'flag' in dict_of_param:
            flag = dict_of_param['flag']
        if 'server' in dict_of_param:
            server = dict_of_param['server']

        log(list, "__main__")

        token = parsed_dict['token']

        result_url = os.path.join(parsed_dict['url'], 'results')
        result_path = os.path.join(result_path, 'results')
        source_type = parsed_dict['source_type']
        db_token = parsed_dict['dropbox_token']

        if parsed_dict['exec'] == 'ImageStitch':
            print "Image Stitching running"
            output = run_executable(list, True, socketid)
            sendsMessageToRedis(userid, jobid, source_type, socketid, output, result_path, result_url,
                                dropbox_token=db_token)
            r.publish('chat', json.dumps({'done': str('Image Stitching done'), 'socketid': str(socketid), 'token': token, 'jobid': jobid}))


        elif(parsed_dict['exec'] == 'VOCRelease5'):
            # output = run_executable(list)
            output = run_matlab_code(mlab_obj, '/var/www/html/cloudcv/voc-dpm-matlab-bridge/pascal_object_detection.m', list, parsed_dict['socketid'])
            sendsMessageToRedis(userid, jobid, source_type, socketid, output, result_path, result_url,
                                dropbox_token=db_token)
            r.publish('chat', json.dumps({'message': str('Bounding Boxes Generated'), 'socketid': str(socketid), 'token': token, 'jobid': jobid}))

        elif(parsed_dict['exec'] == 'classify'):
            run_classification(parsed_dict['userid'], parsed_dict['jobid'], parsed_dict['image_path'],
                               parsed_dict['socketid'], parsed_dict['token'], parsed_dict['source_type'],result_path,
                               db_token=db_token)

        elif(parsed_dict['exec'] == 'features'):
            print result_url
            tags = {}
            matlabfilepath = decaf_cal_feature.calculate_decaf(parsed_dict['image_path'], result_path,3,socketid, tags)
            sendsMessageToRedis(userid, jobid, source_type, socketid, '', result_path, result_url,
                                    dropbox_token=db_token)
            r.publish('chat', json.dumps({'done': str('Features Generated'),
                                          'socketid': str(socketid), 'token': token, 'jobid': jobid}))

    except Exception as e:
        log_and_exit(str(traceback.format_exc()), socketid)
Exemple #5
0
def run(parsed_dict):
    socketid = str(parsed_dict['socketid'])
    try:
        if 'dropbox_token' not in parsed_dict or parsed_dict[
                'dropbox_token'] == 'None':
            parsed_dict['dropbox_token'] = None

        global jobid

        jobid = str(parsed_dict['jobid'])
        userid = str(parsed_dict['userid'])

        image_path = parsed_dict['image_path']
        result_path = str(parsed_dict['result_path'])

        list = None
        flag = None
        server = None

        dict_of_param = createList(image_path, parsed_dict)
        if 'dict' in dict_of_param:
            list = dict_of_param['dict']
        if 'flag' in dict_of_param:
            flag = dict_of_param['flag']
        if 'server' in dict_of_param:
            server = dict_of_param['server']

        log(list, "__main__")

        token = parsed_dict['token']

        result_url = os.path.join(parsed_dict['url'], 'results')
        result_path = os.path.join(result_path, 'results')
        source_type = parsed_dict['source_type']
        db_token = parsed_dict['dropbox_token']

        if parsed_dict['exec'] == 'ImageStitch':
            print "Image Stitching running"
            output = run_executable(list, True, socketid)
            sendsMessageToRedis(userid,
                                jobid,
                                source_type,
                                socketid,
                                output,
                                result_path,
                                result_url,
                                dropbox_token=db_token)
            r.publish(
                'chat',
                json.dumps({
                    'done': str('Image Stitching done'),
                    'socketid': str(socketid),
                    'token': token,
                    'jobid': jobid
                }))

        elif (parsed_dict['exec'] == 'VOCRelease5'):
            # output = run_executable(list)
            output = run_matlab_code(
                mlab_obj,
                '/var/www/html/cloudcv/voc-dpm-matlab-bridge/pascal_object_detection.m',
                list, parsed_dict['socketid'])
            sendsMessageToRedis(userid,
                                jobid,
                                source_type,
                                socketid,
                                output,
                                result_path,
                                result_url,
                                dropbox_token=db_token)
            r.publish(
                'chat',
                json.dumps({
                    'message': str('Bounding Boxes Generated'),
                    'socketid': str(socketid),
                    'token': token,
                    'jobid': jobid
                }))

        elif (parsed_dict['exec'] == 'classify'):
            run_classification(parsed_dict['userid'],
                               parsed_dict['jobid'],
                               parsed_dict['image_path'],
                               parsed_dict['socketid'],
                               parsed_dict['token'],
                               parsed_dict['source_type'],
                               result_path,
                               db_token=db_token)

        elif (parsed_dict['exec'] == 'features'):
            print result_url
            tags = {}
            matlabfilepath = decaf_cal_feature.calculate_decaf(
                parsed_dict['image_path'], result_path, 3, socketid, tags)
            sendsMessageToRedis(userid,
                                jobid,
                                source_type,
                                socketid,
                                '',
                                result_path,
                                result_url,
                                dropbox_token=db_token)
            r.publish(
                'chat',
                json.dumps({
                    'done': str('Features Generated'),
                    'socketid': str(socketid),
                    'token': token,
                    'jobid': jobid
                }))

    except Exception as e:
        log_and_exit(str(traceback.format_exc()), socketid)