Ejemplo n.º 1
0
 def _convert_request_to_dict(self, request):
     '''
     Utility method: transfer all values from GET, then POST to a dict.
     
     Note: uses 'getlist' to retrieve all values:
     - if a value is single valued, then unwrap from the list
     - downstream methods expecting a list value must deal with non-list 
     single values
     Note: does not handle request.FILES
     '''
     DEBUG = False or logger.isEnabledFor(logging.DEBUG)
     
     _dict = {}
     if request is None:
         return _dict
     for key in request.GET.keys():
         val = request.GET.getlist(key)
         if DEBUG:
             logger.info('get key: %r, val: %r', key, val)
         # Jquery Ajax will send array list params with a "[]" suffix
         if '[]' in key and key[-2:] == '[]':
             key = key[:-2]
         if len(val) == 1:
             _dict[key] = val[0]
         else:
             _dict[key] = val
         
     for key in request.POST.keys():
         val = request.POST.getlist(key)
         if DEBUG:
             logger.info('post key: %r, val: %r', key, val)
         # Jquery Ajax will post array list params with a "[]" suffix
         key = key.replace('[]','')
         if len(val) == 1:
             _dict[key] = val[0]
         else:
             _dict[key] = val
     
     # check for single-valued known list values
     # Note: Jquery Ajax will post array list params with a "[]" suffix
     known_list_values = [
         'includes','exact_fields', 'order_by', 'visibilities',
         'other_screens']
     # extend with alternate possible keys (depending on posted format)
     known_list_values.extend(['%s[]'%k for k in known_list_values])
     for key in known_list_values:
         val = _dict.get(key,[])
         if isinstance(val, basestring):
             _dict[key] = [val]
     
     # Parse known boolean params for convenience
     http_boolean_params = [
         HTTP_PARAM_DATA_INTERCHANGE,HTTP_PARAM_RAW_LISTS,
         HTTP_PARAM_USE_VOCAB, HTTP_PARAM_USE_TITLES]
     for key in http_boolean_params:
         _dict[key] = parse_val(
             _dict.get(key, False),key, 'boolean')
     if DEBUG:
         logger.info('params: %r', _dict)
     return _dict    
Ejemplo n.º 2
0
 def _convert_request_to_dict(self, request):
     '''
     Transfer all values from GET, then POST to a dict
     Note: uses 'getlist' to retrieve all values:
     - if a value is single valued, then unwrap from the list
     - downstream methods expecting a list value must deal with non-list single values
     '''
     _dict = {}
     
     for key in request.GET.keys():
         val = request.GET.getlist(key)
         # Jquery Ajax will post array list params with a "[]" suffix - 20151015
         if '[]' in key and key[-2:] == '[]':
             key = key[:-2]
         if len(val) == 1:
             _dict[key] = val[0]
         else:
             _dict[key] = val
         
     for key in request.POST.keys():
         val = request.POST.getlist(key)
         # Jquery Ajax will post array list params with a "[]" suffix - 20151015
         key = key.replace('[]','')
         if len(val) == 1:
             _dict[key] = val[0]
         else:
             _dict[key] = val
     
     # check for single-valued known list values
     # Note: Jquery Ajax will post array list params with a "[]" suffix - 20151015
     known_list_values = [
         'includes','exact_fields', 'order_by', 'visibilities','other_screens',  
         'includes[]', 'order_by[]','exact_fields[]', 'visibilities[]',
         'other_screens[]']
     for key in known_list_values:
         val = _dict.get(key,[])
         if isinstance(val, basestring):
             _dict[key] = [val]
     
     # Parse known boolean params for convenience
     http_boolean_params = [
         HTTP_PARAM_DATA_INTERCHANGE,HTTP_PARAM_RAW_LISTS,
         HTTP_PARAM_USE_VOCAB, HTTP_PARAM_USE_TITLES]
     for key in http_boolean_params:
         _dict[key] = parse_val(
             _dict.get(key, False),key, 'boolean')
     
     return _dict    
Ejemplo n.º 3
0
def parse_columns(columns):
    '''
    Parse the Screen Result input file Data Columns sheet into valid API 
        Data Columns input.
    '''
    parsed_cols = OrderedDict()
    errors = {}
    for i,column in enumerate(columns):
        parsed_col = {
            'is_derived': False,
            'is_follow_up_data': False,
            'ordinal': i
        }
        logger.debug('parsing column: %r', column['data_worksheet_column'])
        if column['data_worksheet_column'] in parsed_cols:
            raise ValidationError(
                key='data_worksheet_column', 
                msg='%r is listed more than once' 
                    % column['data_worksheet_column'])
        parsed_cols[column['data_worksheet_column']] = parsed_col
        for key,val in column.items():
            if key == 'is_follow_up_data':
                parsed_col[key] = ( val and val.lower() == 'follow up')
            elif key == 'data_type':
                val = default_converter(val)
                # handle validation errors in the api
                if val not in DATA_TYPE_VALUES:
                    key = '%s:%s' % (column['data_worksheet_column'],'data_type')
                    errors[key] = 'val: %r must be one of %r' % (val,DATA_TYPE_VALUES)
                parsed_col[key] = val
            elif key == 'assay_readout_type':
                parsed_col[key] = default_converter(val)
            else:
                if key == 'how_derived':
                    parsed_col['is_derived'] = ( 
                        val is not None and val.strip() is not '' )
                parsed_col[key] = val
        
        if parsed_col.get('decimal_places') is not None:
            try:
                key = '%s:%s' % (column['data_worksheet_column'],'data_type')
                column['decimal_places'] = parse_val(
                    column['decimal_places'],key,'integer')
            except ValidationError, e:
                errors.update(e.errors)
        logger.debug('parsed_col: %r', parsed_col)
Ejemplo n.º 4
0
def parse_columns(columns_sheet):
    logger.info('parsing columns sheet: %r', columns_sheet.name)
    columns = data_column_generator(sheet_cols(columns_sheet))
    parsed_cols = OrderedDict()
    errors = {}
    for i,column in enumerate(columns):
        parsed_col = {
            'is_derived': False,
            'is_follow_up_data': False,
            'ordinal': i
        }
        logger.debug('parsing column: %r', column['data_worksheet_column'])
        parsed_cols[column['data_worksheet_column']] = parsed_col
        for key,val in column.items():
            if key == 'is_follow_up_data':
                parsed_col[key] = ( val and val.lower() == 'follow up')
            elif key == 'data_type':
                val = default_converter(val)
                # handle validation errors in the api
                if val not in DATA_TYPES:
                    key = '%s:%s' % (column['data_worksheet_column'],'data_type')
                    errors[key] = 'val: %r must be one of %r' % (val,DATA_TYPES)
                parsed_col[key] = val
            elif key == 'assay_readout_type':
                parsed_col[key] = default_converter(val)
            else:
                if key == 'how_derived':
                    parsed_col['is_derived'] = ( 
                        val is not None and val.strip() is not '' )
                parsed_col[key] = val
        
        if parsed_col.get('decimal_places', None):
            try:
                key = '%s:%s' % (column['data_worksheet_column'],'data_type')
                column['decimal_places'] = parse_val(
                    column['decimal_places'],key,'integer')
            except ValidationError, e:
                errors.update(e.errors)
        logger.debug('parsed_col: %r', parsed_col)
Ejemplo n.º 5
0
    def top_level(self, request, api_name=None):
        '''
        A view that returns a serialized list of all resources registers
        to the API.
        '''
        fullschema = parse_val(
            request.GET.get('fullschema', False),
            'fullschema', 'boolean')

        available_resources = {}

        if api_name is None:
            api_name = self.api_name

        for name, resource in self._registry.items():
            if not fullschema:
                schema = self._build_reverse_url('api_get_schema', kwargs={
                    'api_name': api_name,
                    'resource_name': name,
                })
            else:
                schema = resource.build_schema()

            available_resources[name] = {
                'list_endpoint': 
                    self._build_reverse_url('api_dispatch_list', kwargs={
                        'api_name': api_name,
                        'resource_name': name,
                    }),
                'schema': schema,
            }

        serializer = LimsSerializer()
        content_type = serializer.get_accept_content_type(request)
        serialized =  serializer.serialize(available_resources, content_type)
        return HttpResponse(
            content=serialized, 
            content_type=content_type)
Ejemplo n.º 6
0
    def top_level(self, request, api_name=None):
        '''
        A view that returns a serialized list of all resources registers
        to the API.
        '''
        fullschema = parse_val(request.GET.get('fullschema', False),
                               'fullschema', 'boolean')

        available_resources = {}

        if api_name is None:
            api_name = self.api_name

        for name, resource in self._registry.items():
            if not fullschema:
                schema = self._build_reverse_url('api_get_schema',
                                                 kwargs={
                                                     'api_name': api_name,
                                                     'resource_name': name,
                                                 })
            else:
                schema = resource.build_schema()

            available_resources[name] = {
                'list_endpoint':
                self._build_reverse_url('api_dispatch_list',
                                        kwargs={
                                            'api_name': api_name,
                                            'resource_name': name,
                                        }),
                'schema':
                schema,
            }

        serializer = LimsSerializer()
        content_type = serializer.get_accept_content_type(request)
        serialized = serializer.serialize(available_resources, content_type)
        return HttpResponse(content=serialized, content_type=content_type)
Ejemplo n.º 7
0
def parse_result_row(i,parsed_columns,result_row):    
    '''
    Parse the Screen Result input file format into a valid API input format:
    - Convert plate_number and well_name into a well_id
    - Convert the assay_well_control_type input:
        use the ASSAY_WELL_CONTROL_TYPES to map api schema assaywell.control_type
    - Convert the exclude column specifiers into known column letters:
        "all" is converted to a list of all column letters
    - Parse value columns according to the data_type specified:
        - Create default values for positive columns
        - (TODO: validation rules can be moved to API)
        - Verify that PARTITION_POSITIVE_MAPPING values are used
        - Verify that CONFIRMED_POSITIVE_MAPPING values are used
        - Verify that integer values are integers
        - Verify that decimal values can be parsed as float
    '''
    logger.debug(
        'parse result row: %d, %r:  %r', i, parsed_columns.keys(), result_row)
    
    meta_columns = RESULT_VALUE_FIELD_MAP.values()
    parsed_row = {}
    excluded_cols = []

    well_id_errors = []
    meta_key = 'plate_number'
    val = result_row[meta_key]
    logger.debug('plate value to parse: %r', val)
    plate_number = parse_val(val, meta_key, 'integer')
    if plate_number is None:
        well_id_errors.append('%s is required' % meta_key)
    meta_key = 'well_name'
    val = result_row[meta_key]
    if not val:
        well_id_errors.append('%s is required' % meta_key)
    elif WELL_NAME_PATTERN.match(val):
        wellname = val
    else:
        well_id_errors.append('Well_name val %r does not follow the pattern: %r'
            % (val, WELL_NAME_PATTERN.pattern))
    if well_id_errors:
        raise ParseError(errors={ 'row: %d'%i: well_id_errors })
    
    parsed_row['well_id'] = \
        '%s:%s' % (str(plate_number).zfill(5), wellname)
    
    meta_key = 'assay_well_control_type'
    val = result_row.get(meta_key)
    parsed_row[meta_key] = None
    if val is not None:
        if val.lower() in ASSAY_WELL_CONTROL_TYPES:
            parsed_row[meta_key] = \
                ASSAY_WELL_CONTROL_TYPES[val.lower()]
        else:
            msg = ('%s: val %r is not one of the choices: %r'
                % (meta_key, val, ASSAY_WELL_CONTROL_TYPES))
            logger.error(msg)
            raise ValidationError(key=parsed_row['well_id'], msg=msg)

    meta_key = 'exclude'
    val = result_row.get(meta_key)
    if val is not None:
        if val.lower() == 'all':
            excluded_cols = parsed_columns.keys()
        else:
            excluded_cols = [x.strip().upper() for x in val.split(',')]
            unknown_excluded_cols = (
                set(excluded_cols) - set(parsed_columns.keys()))
            if unknown_excluded_cols:
                raise ValidationError(
                    key = parsed_row['well_id'],
                    msg = 'unknown excluded cols: %r' % unknown_excluded_cols )
            parsed_row[meta_key] = excluded_cols
            
    for colname, raw_val in result_row.items():
        logger.debug('colname: %r, raw_val: %r', colname, raw_val)
        if colname in meta_columns:
            continue
        if colname not in parsed_columns:
            # NOTE: this is no longer an error, as the result value sheet may
            # contain extra columns (selected by user on output)
            logger.debug(
                'result value column %r is not in recognized columns: %r', 
                colname, parsed_columns.keys())
            parsed_row[colname] = raw_val
            continue
        column = parsed_columns[colname]
        if raw_val is None:
            # 20180315 - verified with DJW, default values for
            # positive indicator columns
            if column['data_type'] == DATA_TYPE.BOOLEAN_POSITIVE:
                raw_val = False
            elif column['data_type'] == DATA_TYPE.PARTITIONED_POSITIVE:
                raw_val = 'NP'
            elif column['data_type'] == DATA_TYPE.CONFIRMED_POSITIVE:
                raw_val = 'NT'
            else:
                continue
        
        key = '%s-%s' % (parsed_row['well_id'],colname)
        parsed_row[colname] = raw_val
        
        if column['data_type']  in DATA_TYPE.numeric_types:
            if  column['decimal_places'] > 0:
                # parse, to validate only; use decimal for final parsing
                parse_val(raw_val, key, 'float')
            else:
                parsed_row[colname] = parse_val(raw_val, key, 'integer')
                
        elif column['data_type'] == DATA_TYPE.PARTITIONED_POSITIVE:
            val = raw_val.upper()
            if val not in PARTITION_POSITIVE_MAPPING:
                raise ValidationError(
                    key=key, 
                    msg='val: %r must be one of %r'
                        % (raw_val, PARTITION_POSITIVE_MAPPING.keys()))
            parsed_row[colname] = val
        elif column['data_type'] == DATA_TYPE.CONFIRMED_POSITIVE:
            val = raw_val.upper()
            if val not in CONFIRMED_POSITIVE_MAPPING:
                raise ValidationError(
                    key=key, 
                    msg='val: %r must be one of %r'
                        % (raw_val, CONFIRMED_POSITIVE_MAPPING.keys()))
            parsed_row[colname] = val
        elif column['data_type'] == DATA_TYPE.BOOLEAN_POSITIVE:
            val = parse_val(raw_val,key,'boolean')
            parsed_row[colname] = val
        logger.debug('parsed_row: %r', parsed_row)
    
    return parsed_row
Ejemplo n.º 8
0
def parse_result_row(i,parsed_columns,result_row):    
    
    logger.debug('parse result row: %r', result_row)
    
    meta_columns = RESULT_VALUE_FIELD_MAP.values()
    parsed_row = {}
    excluded_cols = []
    
    meta_key = 'plate_number'
    val = result_row[meta_key]
    logger.debug('plate value to parse: %r', val)
    plate_number = parse_val(val, meta_key, 'integer')
    meta_key = 'well_name'
    val = result_row[meta_key]
    if WELLNAME_MATCHER.match(val):
        wellname = val
    else:
        raise ParseError(
            key=i, 
            msg=('well_name val %r does not follow the pattern: %r'
            % (val, WELLNAME_MATCHER.pattern))) 
    parsed_row['well_id'] = \
        '%s:%s' % (str(plate_number).zfill(5), wellname)
    
    meta_key = 'assay_well_control_type'
    val = result_row.get(meta_key, None)
    parsed_row[meta_key] = None
    if val:
        if val.lower() in ASSAY_WELL_CONTROL_TYPES:
            parsed_row[meta_key] = \
                ASSAY_WELL_CONTROL_TYPES[val.lower()]
        else:
            msg = ('%s: val %r is not one of the choices: %r'
                % (meta_key, val, ASSAY_WELL_CONTROL_TYPES))
            logger.error(msg)
            raise ValidationError(key=parsed_row['well_id'], msg=msg)

    meta_key = 'exclude'
    val = result_row.get(meta_key, None)
    if val:
        if val.lower() == 'all':
            excluded_cols = parsed_columns.keys()
        else:
            excluded_cols = [x.strip().upper() for x in val.split(',')]
            unknown_excluded_cols = (
                set(excluded_cols) - set(parsed_columns.keys()))
            if unknown_excluded_cols:
                raise ValidationError(
                    key = parsed_row['well_id'],
                    msg = 'unknown excluded cols: %r' % unknown_excluded_cols )
            parsed_row[meta_key] = excluded_cols
            
    for colname, raw_val in result_row.items():
        if colname in meta_columns:
            continue
        if colname not in parsed_columns:
            # NOTE: this is no longer an error, as the result value sheet may
            # contain extra columns (selected by user on output)
            logger.debug(
                'result value column %r is not in recognized columns: %r', 
                colname, parsed_columns.keys())
            parsed_row[colname] = raw_val
            continue
        column = parsed_columns[colname]
        if ( column['data_type'] == 'partition_positive_indicator'
            and not raw_val):
            raw_val = 'NP' 
        if ( column['data_type'] == 'confirmed_positive_indicator'
            and not raw_val):
            raw_val = 'NT' 
        if raw_val is None:
            continue
        
        key = '%s-%s' % (parsed_row['well_id'],colname)
        parsed_row[colname] = raw_val
        
        if column['data_type'] == 'numeric':
            if  column['decimal_places'] > 0:
                # parse, to validate
                parse_val(raw_val, key, 'float')
            else:
                parsed_row[colname] = parse_val(raw_val, key, 'integer')
        elif column['data_type'] == 'partition_positive_indicator':
            val = raw_val.upper()
            if val not in PARTITION_POSITIVE_MAPPING:
                raise ValidationError(
                    key=key, 
                    msg='val: %r must be one of %r'
                        % (raw_val, PARTITION_POSITIVE_MAPPING.keys()))
            parsed_row[colname] = val
        elif column['data_type'] == 'confirmed_positive_indicator':
            val = raw_val.upper()
            if val not in CONFIRMED_POSITIVE_MAPPING:
                raise ValidationError(
                    key=key, 
                    msg='val: %r must be one of %r'
                        % (raw_val, CONFIRMED_POSITIVE_MAPPING.keys()))
            parsed_row[colname] = val
        elif column['data_type'] == 'boolean_positive_indicator':
            val = parse_val(raw_val,key,'boolean')
            parsed_row[colname] = val
            
    return parsed_row
Ejemplo n.º 9
0
    def _convert_request_to_dict(self, request):
        '''
        Utility method: transfer all values from GET, then POST to a dict.
        
        Note: uses 'getlist' to retrieve all values:
        - if a value is single valued, then unwrap from the list
        - downstream methods expecting a list value must deal with non-list 
        single values
        Note: does not handle request.FILES
        '''
        DEBUG = False or logger.isEnabledFor(logging.DEBUG)

        _dict = {}
        if request is None:
            return _dict
        for key in request.GET.keys():
            val = request.GET.getlist(key)
            if DEBUG:
                logger.info('get key: %r, val: %r', key, val)
            # Jquery Ajax will send array list params with a "[]" suffix
            if '[]' in key and key[-2:] == '[]':
                key = key[:-2]
            if len(val) == 1:
                _dict[key] = val[0]
            else:
                _dict[key] = val

        for key in request.POST.keys():
            val = request.POST.getlist(key)
            if DEBUG:
                logger.info('post key: %r, val: %r', key, val)
            # Jquery Ajax will post array list params with a "[]" suffix
            key = key.replace('[]', '')
            if len(val) == 1:
                _dict[key] = val[0]
            else:
                _dict[key] = val

        # check for single-valued known list values
        # Note: Jquery Ajax will post array list params with a "[]" suffix
        known_list_values = [
            'includes', 'exact_fields', 'order_by', 'visibilities',
            'other_screens'
        ]
        # extend with alternate possible keys (depending on posted format)
        known_list_values.extend(['%s[]' % k for k in known_list_values])
        for key in known_list_values:
            val = _dict.get(key, [])
            if isinstance(val, basestring):
                _dict[key] = [val]

        # Parse known boolean params for convenience
        http_boolean_params = [
            HTTP_PARAM_DATA_INTERCHANGE, HTTP_PARAM_RAW_LISTS,
            HTTP_PARAM_USE_VOCAB, HTTP_PARAM_USE_TITLES
        ]
        for key in http_boolean_params:
            _dict[key] = parse_val(_dict.get(key, False), key, 'boolean')
        if DEBUG:
            logger.info('params: %r', _dict)
        return _dict