def xls_write_workbook(file, data, request=None, image_keys=None, title_function=None, list_brackets=None): ''' ***WARNING*** xlsx files load fully into memory on display - if there are >~ 2000 images, this will cause performance issues on the client.*** @param sheet_rows iterable of dicts, one per row ''' if not isinstance(data, dict): raise BadRequest( 'unknown data for generic xls serialization: %r' % type(data)) wb = xlsxwriter.Workbook(file, {'constant_memory': True}) logger.info('xls_write_workbook for data: %r', data.keys()) for key, sheet_rows in data.items(): logger.info('type sheet_rows: %r', type(sheet_rows)) if isinstance(sheet_rows, (dict, OrderedDict)): sheet_name = default_converter(key) logger.info('writing sheet %r...', sheet_name) sheet = wb.add_worksheet(sheet_name) for i, row in enumerate(csvutils.dict_to_rows(sheet_rows)): sheet.write_row(i,0,row) elif isinstance(sheet_rows, basestring): sheet_name = default_converter(key) logger.info('writing single string sheet %r...', sheet_name) sheet = wb.add_worksheet(sheet_name) sheet.write_string(0,0,sheet_rows) else: sheet_name = default_converter(key) logger.info('writing sheets for base name %r...', sheet_name) write_rows_to_sheet(wb, sheet_rows, sheet_name, request=request, image_keys=image_keys, title_function=title_function, list_brackets=list_brackets) wb.close()
def generic_xls_write_workbook(file, data): ''' Writes a dict of iterables to a workbook, where key=sheet name, value=iterable that is ready to write (e.g. a list) ''' wb = xlsxwriter.Workbook(file, {'constant_memory': True}) if isinstance(data, dict): logger.info('generic_xls_write_workbook for data: %r', data.keys()) for key, sheet_rows in data.items(): sheet_name = default_converter(key) logger.info('writing sheet %r...', sheet_name) sheet = wb.add_worksheet(sheet_name) if isinstance(sheet_rows, dict): for i, row in enumerate(csvutils.dict_to_rows(sheet_rows)): sheet.write_row(i,0,row) elif isinstance(sheet_rows, basestring): sheet.write_string(0,0,sheet_rows) else: generic_write_rows_to_sheet(sheet_rows, sheet) else: raise BadRequest( 'unknown data for generic xls serialization: %r' % type(data)) logger.info('save to file; %r', file.name) wb.close()
def generic_xls_write_workbook(file, data): ''' Writes a dict of iterables to a workbook, where key=sheet name, value=iterable that is ready to write (e.g. a list) ''' wb = xlsxwriter.Workbook(file, {'constant_memory': True}) if isinstance(data, dict): logger.info('generic_xls_write_workbook for data: %r', data.keys()) for key, sheet_rows in data.items(): sheet_name = default_converter(key) logger.info('writing sheet %r...', sheet_name) sheet = wb.add_worksheet(sheet_name) if isinstance(sheet_rows, dict): for i, row in enumerate(csvutils.dict_to_rows(sheet_rows)): sheet.write_row(i, 0, row) elif isinstance(sheet_rows, basestring): sheet.write_string(0, 0, sheet_rows) else: generic_write_rows_to_sheet(sheet_rows, sheet) else: raise BadRequest('unknown data for generic xls serialization: %r' % type(data)) logger.info('save to file; %r', file.name) wb.close()
def create_attached_file_type_vocab(apps): vocab_file = os.path.join( lims.settings.PROJECT_ROOT, '..', 'db', 'static', 'api_init', 'vocabulary_attachedfiletype_data.csv') logger.info('write vocabularies to %s' % vocab_file) resource_uri = '/reports/api/v1/vocabulary/%s/%s/' with open(vocab_file, 'w') as _file: vocab_writer = csv.writer(_file) header = ['resource_uri', 'key', 'scope', 'ordinal', 'title', 'is_retired'] vocab_writer.writerow(header) _scope = 'attachedfiletype.%s' for i, obj in enumerate(apps.get_model('db', 'AttachedFileType') .objects.all().order_by('value')): key = obj.value.lower() key = key.replace('nsrb','') # For Jen Smith key = default_converter(obj.value) scope = _scope % obj.for_entity_type title = obj.value ordinal = i is_retired = key in [ 'marcus_application','miare_document', 'nerce_screener_supplies_list'] row = [resource_uri % (scope, key), key, scope, ordinal, title, is_retired] vocab_writer.writerow(row) logger.info(str(('created', row))) (apps.get_model('db', 'AttachedFile').objects .filter(attached_file_type=obj).update(type=key)) # For Jen Smith - 20160720 # /reports/api/v1/vocabulary/attachedfiletype.screen/material_transfer_agreement material_transfer_agreement attachedfiletype.screen 21 Material Transfer Agreement # /reports/api/v1/vocabulary/attachedfiletype.screen/other other attachedfiletype.screen 22 Other # /reports/api/v1/vocabulary/attachedfiletype.user/other other attachedfiletype.user 22 Other # row = [resource_uri % (scope, key), key, scope, ordinal, title, is_retired] # vocab_writer.writerow(row) api_init_actions_file = os.path.join( lims.settings.PROJECT_ROOT, '..', 'db', 'static', 'api_init', 'api_init_actions.csv') logger.info('write %s entry to %s' % (vocab_file, api_init_actions_file)) with open(api_init_actions_file, 'a+') as _file: new_row = ['patch', 'vocabulary', os.path.basename(vocab_file)] reader = csv.reader(_file) found = False for row in reader: if row == new_row: found = True break if not found: writer = csv.writer(_file) writer.writerow(new_row) else: logger.info('api_init entry for row already created: %r' % row) logger.info('done')
def xls_write_workbook(file, data, request=None, image_keys=None, title_function=None, list_brackets=None): ''' ***WARNING*** xlsx files load fully into memory on display - if there are >~ 2000 images, this will cause performance issues on the client.*** @param sheet_rows iterable of dicts, one per row ''' if not isinstance(data, dict): raise BadRequest('unknown data for generic xls serialization: %r' % type(data)) wb = xlsxwriter.Workbook(file, {'constant_memory': True}) logger.info('xls_write_workbook for data: %r', data.keys()) for key, sheet_rows in data.items(): logger.info('type sheet_rows: %r', type(sheet_rows)) if isinstance(sheet_rows, (dict, OrderedDict)): sheet_name = default_converter(key) logger.info('writing sheet %r...', sheet_name) sheet = wb.add_worksheet(sheet_name) for i, row in enumerate(csvutils.dict_to_rows(sheet_rows)): sheet.write_row(i, 0, row) elif isinstance(sheet_rows, basestring): sheet_name = default_converter(key) logger.info('writing single string sheet %r...', sheet_name) sheet = wb.add_worksheet(sheet_name) sheet.write_string(0, 0, sheet_rows) else: sheet_name = default_converter(key) logger.info('writing sheets for base name %r...', sheet_name) write_rows_to_sheet(wb, sheet_rows, sheet_name, request=request, image_keys=image_keys, title_function=title_function, list_brackets=list_brackets) wb.close()
def result_value_generator(result_values): logger.info('Write the result values sheet') header_row = [] header_row.extend(RESULT_VALUE_FIELD_MAP.keys()) # TODO: allow column titles to be optional header_row.extend([fields[key].get('title', key) for key in data_columns]) header_row.extend(other_columns) row_count = 0 for result_value in result_values: row_count += 1 row = [] row.extend(result_value['well_id'].split(':')) if ( result_value.has_key('assay_well_control_type') and result_value['assay_well_control_type'] ): control_type = default_converter(result_value['assay_well_control_type']) # note: "empty", "experimental", "buffer" are values that can be # found in this column, due to legacy data entry, but they are # not valid if control_type in control_type_mapping: row.append(control_type_mapping[control_type]) else: row.append(None) else: row.append(None) excluded_cols = [] if result_value.has_key('exclude') and result_value['exclude']: temp = result_value['exclude'] if hasattr(temp, 'split'): temp = temp.split(LIST_DELIMITER_SQL_ARRAY) logger.debug('excluded data_columns: find %r, in %r', temp, data_columns) for data_column_name in temp: excluded_cols.append(get_column_letter( len(RESULT_VALUE_FIELD_MAP)+1 +data_columns.index(data_column_name))) excluded_cols = sorted(excluded_cols) row.append(','.join(excluded_cols)) for j,key in enumerate(data_columns): if result_value.has_key(key): row.append(result_value[key]) else: row.append(None) # append the non-result value columns to the end of the row for j,key in enumerate(other_columns): if result_value.has_key(key): row.append(result_value[key]) if row_count % 10000 == 0: logger.info('wrote %d rows', row_count) yield OrderedDict(zip(header_row,row))
def parse_columns(columns_sheet): logger.info('parsing columns sheet: %r', columns_sheet.name) columns = data_column_generator(sheet_cols(columns_sheet)) parsed_cols = OrderedDict() errors = {} for i,column in enumerate(columns): parsed_col = { 'is_derived': False, 'is_follow_up_data': False, 'ordinal': i } logger.debug('parsing column: %r', column['data_worksheet_column']) parsed_cols[column['data_worksheet_column']] = parsed_col for key,val in column.items(): if key == 'is_follow_up_data': parsed_col[key] = ( val and val.lower() == 'follow up') elif key == 'data_type': val = default_converter(val) # handle validation errors in the api if val not in DATA_TYPES: key = '%s:%s' % (column['data_worksheet_column'],'data_type') errors[key] = 'val: %r must be one of %r' % (val,DATA_TYPES) parsed_col[key] = val elif key == 'assay_readout_type': parsed_col[key] = default_converter(val) else: if key == 'how_derived': parsed_col['is_derived'] = ( val is not None and val.strip() is not '' ) parsed_col[key] = val if parsed_col.get('decimal_places', None): try: key = '%s:%s' % (column['data_worksheet_column'],'data_type') column['decimal_places'] = parse_val( column['decimal_places'],key,'integer') except ValidationError, e: errors.update(e.errors) logger.debug('parsed_col: %r', parsed_col)
def create_serviceactivity_vocab(vocab_writer, attr, scope, query): resource_uri = '/reports/api/v1/vocabulary/%s/%s/' logger.info('create simple vocab: %s, %s', attr, scope) vocabs = [] for ordinal, attr_value in (enumerate( query.values_list(attr, flat=True).distinct(attr).order_by(attr))): if not attr_value: continue key = default_converter(attr_value) title = attr_value _resource_uri = resource_uri % (scope, key) vocabs.append([_resource_uri, key, scope, ordinal, title]) for row in vocabs: title = row[4] key = row[1] # query.filter(**{ '%s__exact' % attr: title }).update(**{ attr: key }) vocab_writer.writerow(row) logger.info('updated vocab: %r' % row)
def migrate_screen_status(apps, schema_editor): ''' Migrate the screen_status_item table to the "status" field of the screen object and create the needed ApiLog entries to record the history NOTE: manual migration 0002 must be run first: - adds an "id" field to screen_status_item - copies the latest status to the screen.status field ''' # Create a history log for all of the status's for each screen, # and store the _current/latest_ status on the new screen.status field count = 0 ScreenStatusItem = apps.get_model('db', 'ScreenStatusItem') for screen in (apps.get_model( 'db', 'Screen').objects.all().order_by('facility_id')): logger.info('process screen: %s', screen.facility_id) if screen.status: # Clean up the vocabulary used in the status_item table # NOTE: migration 0003 shows a newer way of generating the vocabs # - this is ok for these screen.status = default_converter(screen.status) screen.save() # now scan the screen_status_items to recreate logs prev_item = None for status in (ScreenStatusItem.objects.filter( screen=screen).order_by('status_date')): diffs = {} if prev_item: diffs['status'] = [prev_item.status, status.status] else: diffs['status'] = [None, status.status] log = make_log(apps, status.status_date, 'screen', screen.facility_id, diffs, username='******') logger.debug('created log: %d: %r', count, log) prev_item = status count = count + 1 logger.info('updated: %d screen status entries', count)
def create_serviceactivity_vocab(vocab_writer, attr, scope, query): resource_uri = '/reports/api/v1/vocabulary/%s/%s/' logger.info('create simple vocab: %s, %s', attr,scope) vocabs = [] for ordinal, attr_value in (enumerate( query.values_list(attr, flat=True) .distinct(attr).order_by(attr))): if not attr_value: continue key = default_converter(attr_value) title = attr_value _resource_uri = resource_uri % (scope, key) vocabs.append([_resource_uri, key, scope, ordinal, title]) for row in vocabs: title = row[4] key = row[1] # query.filter(**{ '%s__exact' % attr: title }).update(**{ attr: key }) vocab_writer.writerow(row) logger.info('updated vocab: %r' % row)
def data_column_field_mapper(fields): mapped_keys = [] for key in fields: key = key.lower() mapped_key = None if key in DATA_COLUMN_FIELD_MAP: mapped_key = DATA_COLUMN_FIELD_MAP[key] else: for k in DATA_COLUMN_FIELD_MAP.values(): if default_converter(key) == k: mapped_key = k break if not mapped_key: raise ParseError( key=key, msg=('key %r is not in the recognized datacolumn fields: %r' % (key,DATA_COLUMN_FIELD_MAP.keys()))) mapped_keys.append(mapped_key) return mapped_keys
def migrate_screen_status(apps,schema_editor): ''' Migrate the screen_status_item table to the "status" field of the screen object and create the needed ApiLog entries to record the history NOTE: manual migration 0002 must be run first: - adds an "id" field to screen_status_item - copies the latest status to the screen.status field ''' # Create a history log for all of the status's for each screen, # and store the _current/latest_ status on the new screen.status field count=0 ScreenStatusItem = apps.get_model('db','ScreenStatusItem') for screen in ( apps.get_model('db','Screen').objects.all().order_by('facility_id')): logger.info('process screen: %s', screen.facility_id) if screen.status: # Clean up the vocabulary used in the status_item table # NOTE: migration 0003 shows a newer way of generating the vocabs # - this is ok for these screen.status = default_converter(screen.status) screen.save() # now scan the screen_status_items to recreate logs prev_item = None for status in ( ScreenStatusItem.objects.filter(screen=screen) .order_by('status_date')): diffs = {} if prev_item: diffs['status'] = [prev_item.status, status.status] else: diffs['status'] = [None, status.status] log = make_log( apps,status.status_date, 'screen', screen.facility_id, diffs, username='******') logger.debug('created log: %d: %r', count, log) prev_item = status count = count + 1 logger.info('updated: %d screen status entries', count)
def create_user_checklist_items(apps, schema_editor): # prerequisites: # - convert checklist_item / checklist_item_event entries into into # checklistitem.* vocabularies (migration 0003) # - create the user_checklist_item table (0002) ChecklistItem = apps.get_model('db', 'ChecklistItem') UserChecklistItem = apps.get_model('db', 'UserChecklistItem') ci_group_map = {} for obj in ChecklistItem.objects.all().distinct('checklist_item_group'): key = default_converter(obj.checklist_item_group) ci_group_map[obj.checklist_item_group] = key ci_name_map = {} for obj in ChecklistItem.objects.all().distinct('item_name'): key = default_converter(obj.item_name) ci_name_map[obj.item_name] = key # create entries in the user_checklist_item table # note: status values are hard-coded to correspond to the vocabulary # keys (created in migration 0002) sql_keys = [ 'suid', 'cigroup', 'ciname', 'su_username', 'admin_username', 'admin_suid', 'admin_upid', 'date_performed', 'date_created', 'status', 'is_notified' ] sql = ''' select screening_room_user_id, ci.checklist_item_group, ci.item_name, su.username su_username, admin.username admin_username, admin.screensaver_user_id admin_suid, up.id admin_upid, cie.date_performed, cie.date_created, case when cie.is_not_applicable then 'n_a' when ci.is_expirable and cie.date_performed is not null then case when cie.is_expiration then 'deactivated' else 'activated' end when cie.date_performed is not null then 'completed' else 'not_completed' end as status, ( select 1 from screening_room_user sru where sru.last_notified_smua_checklist_item_event_id = cie.checklist_item_event_id UNION select 1 from screening_room_user sru where sru.last_notified_rnaiua_checklist_item_event_id = cie.checklist_item_event_id ) as is_notified from checklist_item ci join checklist_item_event cie using(checklist_item_id) join screensaver_user su on screening_room_user_id=su.screensaver_user_id join screensaver_user admin on cie.created_by_id=admin.screensaver_user_id left join reports_userprofile up on up.id=admin.user_id order by screening_room_user_id, checklist_item_group, item_name, cie.date_performed asc; ''' connection = schema_editor.connection cursor = connection.cursor() log_ref_resource_name = 'userchecklistitem' _dict = None log = None uci_hash = {} unique_log_keys = set() try: cursor.execute(sql) i = 0 for row in cursor: _dict = dict(zip(sql_keys, row)) key = '/'.join( [str(_dict['suid']), _dict['cigroup'], _dict['ciname']]) previous_dict = uci_hash.get(key) logger.debug('previous_dict: %s:%s' % (key, previous_dict)) date_time = pytz.timezone('US/Eastern').localize( _dict['date_created']) if date_time.date() != _dict['date_performed']: # only use the less accurate date_performed date if that date # is not equal to the date_created date date_time = pytz.timezone('US/Eastern').localize( datetime.datetime.combine(_dict['date_performed'], datetime.datetime.min.time())) if previous_dict: uci = previous_dict['obj'] uci.admin_user_id = int(_dict['admin_suid']) uci.status = _dict['status'] uci.previous_status = previous_dict['status'] if (previous_dict['is_notified']): # notified date will be this event - 60 days (smua/rnaiua) uci.status_notified_date = (_dict['date_performed'] - datetime.timedelta(days=60)) uci.status_date = _dict['date_performed'] logger.debug( 'saving, dict: %s, prev_dict: %s, status date %s, status_notified: %s', _dict, previous_dict, uci.status_date, uci.status_notified_date) uci.save() logger.debug('update uci: %s,%s,%s,%s', uci.status, uci.status_date, uci.previous_status, uci.status_notified_date) else: uci_hash[key] = _dict logger.debug( str(('create user checklist item', _dict, _dict['date_performed'].isoformat()))) uci = UserChecklistItem.objects.create( screensaver_user_id=int(_dict['suid']), admin_user_id=int(_dict['admin_suid']), item_group=ci_group_map[_dict['cigroup']], item_name=ci_name_map[_dict['ciname']], status=_dict['status'], status_date=_dict['date_performed']) uci.save() _dict['obj'] = uci logger.debug('created uci: %s,%s,%s', uci.status, uci.status_date) i += 1 # create the apilog for this item log = ApiLog() log.ref_resource_name = log_ref_resource_name log.key = '/'.join( [_dict['su_username'], uci.item_group, uci.item_name]) log.username = _dict['admin_username'] log.user_id = _dict['admin_upid'] log.date_time = date_time log.api_action = 'PATCH' log.uri = '/'.join([log.ref_resource_name, log.key]) log.comment = 'status=%s' % _dict['status'] # is the key (date_time, actually) unique? full_key = '/'.join( [log.ref_resource_name, log.key, str(log.date_time)]) while full_key in unique_log_keys: # add a second to make it unique; because date performed is a date, logger.info(str(('time collision for: ', full_key))) log.date_time = log.date_time + datetime.timedelta(0, 1) full_key = '/'.join( [log.ref_resource_name, log.key, str(log.date_time)]) unique_log_keys.add(full_key) if previous_dict: diff_keys = ['status'] diffs = {} logger.debug(str(('found previous_dict', previous_dict))) diff_keys.append('admin_username') diffs['admin_username'] = [ previous_dict['admin_username'], _dict['admin_username'] ] diff_keys.append('status_date') diffs['status_date'] = [ previous_dict['date_performed'].isoformat(), _dict['date_performed'].isoformat() ] diffs['status'] = [previous_dict['status'], _dict['status']] diff_keys.append('previous_status') diffs['previous_status'] = [None, previous_dict['status']] log.diff_keys = json.dumps(diff_keys) log.diffs = json.dumps(diffs) logger.debug('create log: %s', log) log.save() log = None if i % 1000 == 0: logger.info(str(('created', i, 'logs'))) except Exception, e: logger.exception('migration exc') raise e
def create_checklist_vocabularies(apps): # create a separate vocab file: checklist_item_vocab, add to api_init.csv # output vocabs into a vocabulary patch file vocab_file = os.path.join(PROJECT_ROOT, '..', 'db', 'static', 'api_init', 'vocabulary_checklists_data.csv') logger.info('write vocabularies to %s' % vocab_file) resource_uri = '/reports/api/v1/vocabulary/%s/%s/' with open(vocab_file, 'w') as _file: vocab_writer = csv.writer(_file) header = [ 'resource_uri', 'key', 'scope', 'ordinal', 'title', 'expire_interval_days', 'expire_notifiy_days', '' ] vocab_writer.writerow(header) # ci_group_map = {} scope = 'checklistitem.group' default_order = [ 'mailing', 'forms', 'non-harvard', 'imaging', 'legacy' ] for obj in (apps.get_model( 'db', 'ChecklistItem').objects.all().distinct('checklist_item_group') ): key = default_converter(obj.checklist_item_group) ordinal = 0 for i, x in enumerate(default_order): if x in obj.checklist_item_group.lower(): ordinal = i break title = obj.checklist_item_group row = [ resource_uri % (scope, key), key, scope, ordinal, title, None, None ] vocab_writer.writerow(row) # ci_group_map[obj.checklist_item_group] = key logger.info(str(('created', row))) _scope = 'checklistitem.%s.name' # ci_name_map = {} for obj in (apps.get_model( 'db', 'ChecklistItem').objects.all().distinct('item_name')): key = default_converter(obj.item_name) scope = _scope % default_converter(obj.checklist_item_group) # NOTE: fore user_checklist_item overload of vocabularies: if key in ('current_rnai_user_agreement_active', 'current_small_molecule_user_agreement_active'): expire_interval_days = 720 expire_notifiy_days = 60 else: expire_interval_days = None expire_notifiy_days = None title = obj.item_name ordinal = obj.order_statistic row = [ resource_uri % (scope, key), key, scope, ordinal, title, expire_interval_days, expire_notifiy_days ] vocab_writer.writerow(row) # ci_name_map[obj.item_name] = key logger.info(str(('created', row))) scope = 'checklistitem.status' status_values = [ { 'key': 'not_completed', 'title': 'Not Completed', 'ordinal': 0 }, { 'key': 'activated', 'title': 'Activated', 'ordinal': 1 }, { 'key': 'deactivated', 'title': 'Deactivated', 'ordinal': 2 }, { 'key': 'na', 'title': 'N/A', 'ordinal': 3 }, { 'key': 'completed', 'title': 'Completed', 'ordinal': 4 }, ] for _dict in status_values: _dict['scope'] = scope row = [ resource_uri % (_dict['scope'], _dict['key']), _dict['key'], _dict['scope'], _dict['ordinal'], _dict['title'], None, None ] vocab_writer.writerow(row) logger.info(str(('created', row))) api_init_actions_file = os.path.join(lims.settings.PROJECT_ROOT, '..', 'db', 'static', 'api_init', 'api_init_actions.csv') logger.info('write %s entry to %s' % (vocab_file, api_init_actions_file)) with open(api_init_actions_file, 'a+') as _file: new_row = ['patch', 'vocabulary', os.path.basename(vocab_file)] reader = csv.reader(_file) found = False for row in reader: if row == new_row: found = True break if not found: logger.info('write api_init row: %s' % new_row) writer = csv.writer(_file) writer.writerow(new_row) else: logger.info('api_init entry for row already created: %r' % row) logger.info('vocabulary creation done')
def create_attached_file_type_vocab(apps): vocab_file = os.path.join(lims.settings.PROJECT_ROOT, '..', 'db', 'static', 'api_init', 'vocabulary_attachedfiletype_data.csv') logger.info('write vocabularies to %s' % vocab_file) resource_uri = '/reports/api/v1/vocabulary/%s/%s/' with open(vocab_file, 'w') as _file: vocab_writer = csv.writer(_file) header = [ 'resource_uri', 'key', 'scope', 'ordinal', 'title', 'is_retired' ] vocab_writer.writerow(header) _scope = 'attachedfiletype.%s' for i, obj in enumerate( apps.get_model( 'db', 'AttachedFileType').objects.all().order_by('value')): key = obj.value.lower() key = key.replace('nsrb', '') # For Jen Smith key = default_converter(obj.value) scope = _scope % obj.for_entity_type title = obj.value ordinal = i is_retired = key in [ 'marcus_application', 'miare_document', 'nerce_screener_supplies_list' ] row = [ resource_uri % (scope, key), key, scope, ordinal, title, is_retired ] vocab_writer.writerow(row) logger.info(str(('created', row))) (apps.get_model('db', 'AttachedFile').objects.filter( attached_file_type=obj).update(type=key)) # For Jen Smith - 20160720 # /reports/api/v1/vocabulary/attachedfiletype.screen/material_transfer_agreement material_transfer_agreement attachedfiletype.screen 21 Material Transfer Agreement # /reports/api/v1/vocabulary/attachedfiletype.screen/other other attachedfiletype.screen 22 Other # /reports/api/v1/vocabulary/attachedfiletype.user/other other attachedfiletype.user 22 Other # row = [resource_uri % (scope, key), key, scope, ordinal, title, is_retired] # vocab_writer.writerow(row) api_init_actions_file = os.path.join(lims.settings.PROJECT_ROOT, '..', 'db', 'static', 'api_init', 'api_init_actions.csv') logger.info('write %s entry to %s' % (vocab_file, api_init_actions_file)) with open(api_init_actions_file, 'a+') as _file: new_row = ['patch', 'vocabulary', os.path.basename(vocab_file)] reader = csv.reader(_file) found = False for row in reader: if row == new_row: found = True break if not found: writer = csv.writer(_file) writer.writerow(new_row) else: logger.info('api_init entry for row already created: %r' % row) logger.info('done')
def create_lab_affiliation_vocab(apps): # populate the title field, change the name field to a key replace_phrases = [['harvard medical school', 'hms'], ['harvard university', 'harvard'], ['European Molecular Biology Laboratory', 'embl'], ['[embl]', ''], ['Dana Farber Cancer Institute', 'dfci'], ['University of California', 'uc'], ['University of Massachusetts', 'umass'], ['Institute of Chemistry and Cell Biology', 'iccb'], ['Beth Israel Deaconess Medical Center', 'bidmc'], ['Tufts University', 'tufts'], ['University of California', 'UC'], ['University of Massachusetts', 'umass'], ['[NYU]', ''], ['the', ''], ['of', ''], ['in', ''], ["women's", 'womens'], ["children's", 'childrens']] replace_phrases = [[re.compile(r'\b%s\b' % x, re.IGNORECASE), y] for [x, y] in replace_phrases] vocab_file = os.path.join(lims.settings.PROJECT_ROOT, '..', 'db', 'static', 'api_init', 'vocabulary_lab_affiliations_data.csv') logger.info('write vocabularies to %s' % vocab_file) resource_uri = 'vocabulary/%s/%s/' with open(vocab_file, 'a+') as _file: header = [ 'resource_uri', 'key', 'scope', 'ordinal', 'title', 'comment' ] reader = csv.reader(_file) vocab_writer = csv.writer(_file) defined_vocabs = {} try: header = reader.next() logger.info('read header: %s', header) for row in reader: defined_vocabs[row[4]] = dict(zip(header, row)) except StopIteration as e: logger.info('no entries in %s, writing a new file', vocab_file) vocab_writer.writerow(header) scope = 'labaffiliation.category' default_order = [ 'hms', 'hms_affiliated_hospital', 'hsph', 'harvard_fas', 'broad_icg', 'other' ] for i, la in enumerate( apps.get_model('db', 'LabAffiliation').objects.all().distinct( 'affiliation_category')): if la.affiliation_category not in defined_vocabs: key = default_converter(la.affiliation_category) if key in default_order: ordinal = default_order.index(key) else: ordinal = i + len(default_order) title = la.affiliation_category row = [ resource_uri % (scope, key), key, scope, ordinal, title, la.affiliation_category ] vocab_writer.writerow(row) defined_vocabs[la.affiliation_category] = dict(zip( header, row)) logger.debug('created %s', row) else: logger.info('vocabulary already exists: %s - %s', la.affiliation_category, defined_vocabs[la.affiliation_category]) _scope = 'labaffiliation.category.%s' count_updated = 0 for i, la in enumerate( apps.get_model('db', 'LabAffiliation').objects.all().order_by( 'affiliation_category', 'affiliation_name')): if la.affiliation_name not in defined_vocabs: name = la.affiliation_name.lower() for replacer, replacement in replace_phrases: logger.info('replacer: %s, replacement: %s, name: %s', str(replacer), replacement, name) name = replacer.sub(replacement.lower(), name) logger.info('new name: %s', name) title = la.affiliation_name key = default_converter(name) scope = _scope % default_converter(la.affiliation_category) ordinal = len(defined_vocabs) + 1 row = [ resource_uri % (scope, key), key, scope, ordinal, title, la.affiliation_category ] defined_vocabs[la.affiliation_name] = dict(zip(header, row)) vocab_writer.writerow(row) logger.debug('created row: %s', row) else: logger.info('vocabulary already exists: %s - %s', la.affiliation_name, defined_vocabs[la.affiliation_name]) # now set the screensaveruser field ScreensaverUser = apps.get_model('db', 'ScreensaverUser') if la.labhead_set.exists(): for lh in la.labhead_set.all(): su = ScreensaverUser.objects.get( screensaver_user_id=lh.screensaver_user_id) new_value = defined_vocabs[la.affiliation_name]['key'] logger.debug('updating user %s, lab_affiliation: %s', su.username, new_value) su.lab_head_affiliation = new_value su.save() count_updated += 1 logger.info( 'labaffiliation vocabulary creation done, updated: %s users, %s vocabs', count_updated, len(defined_vocabs))
def create_plate_activity_logs(apps, schema_editor): logger.info('create plate activity logs') Activity = apps.get_model('db', 'Activity') cols = OrderedDict({ 'activity_id': 'a.activity_id', 'username': '******', 'screensaver_user_id': 'screensaver_user_id', 'date_of_activity': 'date_of_activity', 'comments': 'a.comments', 'plate_number': 'plate_number', 'copy_name': 'copy.name', 'library_short_name': 'library.short_name', }) colkeys = cols.keys() _cols = ', '.join([ '%s as %s' % (value,key) for key, value in cols.items() ]) sql = ( 'select ' + _cols + ''' from activity a join screensaver_user on(performed_by_id=screensaver_user_id) join plate on (activity_id=plate.plated_activity_id) join copy using(copy_id) join library using(library_id); ''' ) connection = schema_editor.connection cursor = connection.cursor() cursor.execute(sql) _list = cursor.fetchall() if len(_list) == 0: raise Exception('No plate plated_activities found with sql: %r' % sql) for i,_data in enumerate(_list): _activity = dict(zip(colkeys, _data)) log = ApiLog() log.ref_resource_name = librarycopyplate_resource_name log.key = '/'.join([ _activity['library_short_name'],_activity['copy_name'], str(int(_activity['plate_number'])).zfill(5)]) log.uri = '/'.join([base_uri,log.ref_resource_name,log.key]) log.comment = _activity['comments'] log.date_time = create_log_time(_activity['date_of_activity']) log.username = _activity['username'] log.user_id = _activity['screensaver_user_id'] if "'available'" in log.comment.lower(): log.diffs = { 'status': ['not_specied','available']} elif "'not available'" in log.comment.lower(): log.diffs = { 'status': ['not_specied','not_available']} else: raise Exception('unknown plate.plated_activity comment: %r', _activity) log.save() if i % 1000 == 0: logger.info('processed %d plate plated activity logs', i) logger.info('processed %d plate plated activity logs', i) sql = ( 'select ' + _cols + ''' from activity a join screensaver_user on(performed_by_id=screensaver_user_id) join plate on (activity_id=plate.retired_activity_id) join copy using(copy_id) join library using(library_id); ''' ) cursor.execute(sql) _list = cursor.fetchall() status_change_pattern = re.compile(r".*from '([^\']+)'.*to '([^\']+)'.*") if len(_list) == 0: raise Exception('No plate retired_activities found with sql: %r' % sql) status_terms_recognized = set() for i,_data in enumerate(_list): _activity = dict(zip(colkeys, _data)) log = ApiLog() log.ref_resource_name = librarycopyplate_resource_name log.key = '/'.join([ _activity['library_short_name'],_activity['copy_name'], str(int(_activity['plate_number'])).zfill(5)]) log.uri = '/'.join([base_uri,log.ref_resource_name,log.key]) log.comment = _activity['comments'] log.date_time = create_log_time(_activity['date_of_activity']) log.username = _activity['username'] log.user_id = _activity['screensaver_user_id'] match = status_change_pattern.match(log.comment) if not match: raise Exception('unknown plate.retired_activity comment: %r', _activity) log.diffs = {'status': [ default_converter(match.group(1)), default_converter(match.group(2))]} log.save() status_terms_recognized.add(default_converter(match.group(1))) status_terms_recognized.add(default_converter(match.group(2))) if i % 1000 == 0: logger.info('processed %d plate retired activity logs', i) logger.info('processed %d plate retired activity logs', i) logger.info('status terms recognized: %r', status_terms_recognized)
def create_lab_affiliation_vocab(apps): # populate the title field, change the name field to a key replace_phrases = [ ['harvard medical school', 'hms'], ['harvard university', 'harvard'], ['European Molecular Biology Laboratory', 'embl'], ['[embl]',''], ['Dana Farber Cancer Institute', 'dfci'], ['University of California', 'uc'], ['University of Massachusetts', 'umass'], ['Institute of Chemistry and Cell Biology', 'iccb'], ['Beth Israel Deaconess Medical Center', 'bidmc'], ['Tufts University', 'tufts'], ['University of California', 'UC'], ['University of Massachusetts', 'umass'], ['[NYU]', ''], ['the',''], ['of',''], ['in',''], ["women's", 'womens'], ["children's", 'childrens'] ] replace_phrases = [[re.compile(r'\b%s\b' % x, re.IGNORECASE),y] for [x,y] in replace_phrases ] vocab_file = os.path.join( lims.settings.PROJECT_ROOT, '..', 'db', 'static', 'api_init', 'vocabulary_lab_affiliations_data.csv') logger.info('write vocabularies to %s' % vocab_file) resource_uri = 'vocabulary/%s/%s/' with open(vocab_file, 'a+') as _file: header = ['resource_uri', 'key', 'scope', 'ordinal', 'title', 'comment'] reader = csv.reader(_file) vocab_writer = csv.writer(_file) defined_vocabs = {} try: header = reader.next() logger.info('read header: %s', header) for row in reader: defined_vocabs[row[4]] = dict(zip(header,row)) except StopIteration as e: logger.info('no entries in %s, writing a new file',vocab_file) vocab_writer.writerow(header) scope = 'labaffiliation.category' default_order = ['hms','hms_affiliated_hospital','hsph', 'harvard_fas','broad_icg','other'] for i,la in enumerate(apps.get_model('db', 'LabAffiliation') .objects.all().distinct('affiliation_category')): if la.affiliation_category not in defined_vocabs: key = default_converter(la.affiliation_category) if key in default_order: ordinal = default_order.index(key) else: ordinal = i + len(default_order) title = la.affiliation_category row = [resource_uri % (scope, key), key, scope, ordinal, title, la.affiliation_category ] vocab_writer.writerow(row) defined_vocabs[la.affiliation_category] = dict(zip(header,row)) logger.debug('created %s', row) else: logger.info('vocabulary already exists: %s - %s', la.affiliation_category, defined_vocabs[la.affiliation_category]) _scope = 'labaffiliation.category.%s' count_updated = 0 for i,la in enumerate(apps.get_model('db', 'LabAffiliation') .objects.all() .order_by('affiliation_category','affiliation_name')): if la.affiliation_name not in defined_vocabs: name = la.affiliation_name.lower() for replacer,replacement in replace_phrases: logger.info('replacer: %s, replacement: %s, name: %s', str(replacer),replacement,name) name = replacer.sub(replacement.lower(),name) logger.info('new name: %s', name) title = la.affiliation_name key = default_converter(name) scope = _scope % default_converter(la.affiliation_category) ordinal = len(defined_vocabs) + 1 row = [resource_uri % (scope, key), key, scope, ordinal, title, la.affiliation_category ] defined_vocabs[la.affiliation_name] = dict(zip(header,row)) vocab_writer.writerow(row) logger.debug('created row: %s', row) else: logger.info('vocabulary already exists: %s - %s', la.affiliation_name, defined_vocabs[la.affiliation_name]) # now set the screensaveruser field ScreensaverUser = apps.get_model('db','ScreensaverUser') if la.labhead_set.exists(): for lh in la.labhead_set.all(): su = ScreensaverUser.objects.get(screensaver_user_id=lh.screensaver_user_id) new_value = defined_vocabs[la.affiliation_name]['key'] logger.debug('updating user %s, lab_affiliation: %s', su.username,new_value ) su.lab_head_affiliation = new_value; su.save() count_updated += 1 logger.info('labaffiliation vocabulary creation done, updated: %s users, %s vocabs', count_updated, len(defined_vocabs))
def get_xls_response( data, output_filename,request=None,image_keys=None, title_function=None, list_brackets=None): ''' Create an xlsx file that will be streamed through the StreamingHttpResponse. - if length exceeds MAX_ROWS_PER_XLS_FILE, create multiple files and zip them. - TODO: when using xlsx, can simply add extra sheets to the file. @param output_filename - for naming temp files FIXME: wrap cursor with cursorgenerator; pass in the image columns as arg FIXME: rework this using the generic_xlsx_response as a template: - this method is used for all xlsx serialization at this time, except for in testing, and in ScreenResultSerializer - 20160419. ''' if not isinstance(data, dict): raise BadRequest( 'unknown data for xls serialization: %r, must be a dict of ' 'sheet_row entries' % type(data)) # create a temp dir # with TemporaryFile() as f: temp_dir = os.path.join( settings.TEMP_FILE_DIR, str(time.clock()).replace('.', '_')) os.mkdir(temp_dir) try: # Create an new Excel file and add a worksheet. filename = '%s.xlsx' % (output_filename) temp_file = os.path.join(temp_dir, filename) file_names_to_zip = [temp_file] if DEBUG_STREAMING: logger.info('temp file: %r', temp_file) workbook = xlsxwriter.Workbook(temp_file, {'constant_memory': True}) for key, sheet_rows in data.items(): logger.info('type sheet_rows: %r', type(sheet_rows)) if isinstance(sheet_rows, (dict, OrderedDict)): sheet_name = default_converter(key) logger.info('writing sheet %r...', sheet_name) sheet = workbook.add_worksheet(sheet_name) for i, row in enumerate(csvutils.dict_to_rows(sheet_rows)): sheet.write_row(i,0,row) elif isinstance(sheet_rows, basestring): sheet_name = default_converter(key) logger.info('writing single string sheet %r...', sheet_name) sheet = workbook.add_worksheet(sheet_name) sheet.write_string(0,0,sheet_rows) else: sheet_name = default_converter(key) logger.info('writing sheets for base name %r...', sheet_name) max_rows_per_sheet = 2**20 sheet = workbook.add_worksheet(sheet_name) filerow = 0 sheets = 1 for row,values in enumerate(sheet_rows): if filerow == 0: for i,(key,val) in enumerate(values.items()): title = key if title_function: title = title_function(key) sheet.write_string(filerow,i,title) filerow += 1 for i, (key,val) in enumerate(values.items()): val = csvutils.csv_convert( val, delimiter=LIST_DELIMITER_XLS, list_brackets=list_brackets) if val is not None: if len(val) > 32767: logger.error('warn, row too long, %d, key: %r, len: %d', row,key,len(val) ) if image_keys and key in image_keys: max_rows_per_sheet = MAX_IMAGE_ROWS_PER_XLS_FILE if not request: raise Exception( 'must specify the request parameter for image export') # hack to speed things up: if ( key == 'structure_image' and 'library_well_type' in values and values['library_well_type'].lower() == 'empty' ): continue write_xls_image(sheet, filerow, i, val, request) else: sheet.write_string(filerow,i,val) filerow += 1 if row % 10000 == 0: logger.info('wrote %d rows to temp file', row) if filerow > max_rows_per_sheet: workbook.close() logger.info('wrote file: %r', temp_file) # Create an new Excel file and add a worksheet. filename = '%s_%s.xlsx' % (output_filename, filerow) temp_file = os.path.join(temp_dir, filename) workbook = xlsxwriter.Workbook(temp_file, {'constant_memory': True}) sheet = workbook.add_worksheet(sheet_name) file_names_to_zip.append(temp_file) filerow = 0 workbook.close() logger.info('wrote file: %r', temp_file) content_type = '%s; charset=utf-8' % XLSX_MIMETYPE if len(file_names_to_zip) >1: # create a temp zip file content_type='application/zip; charset=utf-8' temp_file = os.path.join('/tmp',str(time.clock())) logger.info('temp ZIP file: %r', temp_file) with ZipFile(temp_file, 'w') as zip_file: for _file in file_names_to_zip: zip_file.write(_file, os.path.basename(_file)) logger.info('wrote file %r', temp_file) filename = '%s.zip' % output_filename _file = file(temp_file) logger.info('download tmp file: %r, %r',temp_file,_file) wrapper = FileWrapper(_file) response = StreamingHttpResponse( wrapper, content_type=content_type) response['Content-Length'] = os.path.getsize(temp_file) response['Content-Disposition'] = \ 'attachment; filename=%s' % filename return response except Exception, e: logger.exception('xls streaming error') raise e
def create_user_checklist_items(apps, schema_editor): # prerequisites: # - convert checklist_item / checklist_item_event entries into into # checklistitem.* vocabularies (migration 0003) # - create the user_checklist_item table (0002) ChecklistItem = apps.get_model('db','ChecklistItem') UserChecklistItem = apps.get_model('db','UserChecklistItem') ci_group_map = {} for obj in ChecklistItem.objects.all().distinct('checklist_item_group'): key = default_converter(obj.checklist_item_group) ci_group_map[obj.checklist_item_group] = key ci_name_map = {} for obj in ChecklistItem.objects.all().distinct('item_name'): key = default_converter(obj.item_name) ci_name_map[obj.item_name] = key # create entries in the user_checklist_item table # note: status values are hard-coded to correspond to the vocabulary # keys (created in migration 0002) sql_keys = [ 'suid','cigroup','ciname', 'su_username','admin_username','admin_suid','admin_upid', 'date_performed', 'date_created','status','is_notified' ] sql = ''' select screening_room_user_id, ci.checklist_item_group, ci.item_name, su.username su_username, admin.username admin_username, admin.screensaver_user_id admin_suid, up.id admin_upid, cie.date_performed, cie.date_created, case when cie.is_not_applicable then 'n_a' when ci.is_expirable and cie.date_performed is not null then case when cie.is_expiration then 'deactivated' else 'activated' end when cie.date_performed is not null then 'completed' else 'not_completed' end as status, ( select 1 from screening_room_user sru where sru.last_notified_smua_checklist_item_event_id = cie.checklist_item_event_id UNION select 1 from screening_room_user sru where sru.last_notified_rnaiua_checklist_item_event_id = cie.checklist_item_event_id ) as is_notified from checklist_item ci join checklist_item_event cie using(checklist_item_id) join screensaver_user su on screening_room_user_id=su.screensaver_user_id join screensaver_user admin on cie.created_by_id=admin.screensaver_user_id left join reports_userprofile up on up.id=admin.user_id order by screening_room_user_id, checklist_item_group, item_name, cie.date_performed asc; ''' connection = schema_editor.connection cursor = connection.cursor() log_ref_resource_name = 'userchecklistitem' _dict = None log = None uci_hash = {} unique_log_keys = set() try: cursor.execute(sql) i = 0 for row in cursor: _dict = dict(zip(sql_keys,row)) key = '/'.join([str(_dict['suid']),_dict['cigroup'],_dict['ciname']]) previous_dict = uci_hash.get(key) logger.debug('previous_dict: %s:%s' % (key,previous_dict)) date_time = pytz.timezone('US/Eastern').localize(_dict['date_created']) if date_time.date() != _dict['date_performed']: # only use the less accurate date_performed date if that date # is not equal to the date_created date date_time = pytz.timezone('US/Eastern').localize( datetime.datetime.combine( _dict['date_performed'], datetime.datetime.min.time())) if previous_dict: uci = previous_dict['obj'] uci.admin_user_id = int(_dict['admin_suid']) uci.status = _dict['status'] uci.previous_status = previous_dict['status'] if(previous_dict['is_notified']): # notified date will be this event - 60 days (smua/rnaiua) uci.status_notified_date = ( _dict['date_performed'] - datetime.timedelta(days=60)) uci.status_date = _dict['date_performed'] logger.debug('saving, dict: %s, prev_dict: %s, status date %s, status_notified: %s', _dict, previous_dict, uci.status_date, uci.status_notified_date) uci.save() logger.debug('update uci: %s,%s,%s,%s', uci.status,uci.status_date,uci.previous_status,uci.status_notified_date) else: uci_hash[key] = _dict logger.debug(str(('create user checklist item', _dict, _dict['date_performed'].isoformat()))) uci = UserChecklistItem.objects.create( screensaver_user_id = int(_dict['suid']), admin_user_id = int(_dict['admin_suid']), item_group = ci_group_map[_dict['cigroup']], item_name = ci_name_map[_dict['ciname']], status = _dict['status'], status_date = _dict['date_performed']) uci.save() _dict['obj'] = uci logger.debug('created uci: %s,%s,%s', uci.status, uci.status_date) i += 1 # create the apilog for this item log = ApiLog() log.ref_resource_name = log_ref_resource_name log.key = '/'.join([_dict['su_username'],uci.item_group,uci.item_name]) log.username = _dict['admin_username'] log.user_id = _dict['admin_upid'] log.date_time = date_time log.api_action = 'PATCH' log.uri = '/'.join([log.ref_resource_name,log.key]) log.comment = 'status=%s' % _dict['status'] # is the key (date_time, actually) unique? full_key = '/'.join([log.ref_resource_name,log.key,str(log.date_time)]) while full_key in unique_log_keys: # add a second to make it unique; because date performed is a date, logger.info(str(('time collision for: ',full_key))) log.date_time = log.date_time + datetime.timedelta(0,1) full_key = '/'.join([log.ref_resource_name,log.key,str(log.date_time)]) unique_log_keys.add(full_key) if previous_dict: diff_keys = ['status'] diffs = {} logger.debug(str(('found previous_dict', previous_dict))) diff_keys.append('admin_username') diffs['admin_username'] = [previous_dict['admin_username'], _dict['admin_username']] diff_keys.append('status_date') diffs['status_date'] = [ previous_dict['date_performed'].isoformat(), _dict['date_performed'].isoformat()] diffs['status'] = [previous_dict['status'],_dict['status']] diff_keys.append('previous_status') diffs['previous_status'] = [ None, previous_dict['status']] log.diff_keys = json.dumps(diff_keys) log.diffs = json.dumps(diffs) logger.debug('create log: %s', log) log.save() log = None if i%1000 == 0: logger.info(str(('created', i, 'logs'))) except Exception, e: logger.exception('migration exc') raise e
# delete this inconsistent user: # 866 | 14 | 2007-05-24 00:00:00-04 | Ernebjerg | Morten | [email protected] | 617-432-6392 | | using wellmate | | | me44 | 70572885 | | | | | | | | me44 ssuid = 866 try: obj = ScreensaverUser.objects.get(screensaver_user_id=ssuid) obj.delete() except Exception,e: logger.error(str(('cannot find/delete screensaver_user_id', ssuid, e))) # remove the second jgq10 erroneous account ssuid = 3166 try: su = ScreensaverUser.objects.get(screensaver_user_id=ssuid) username = '******' % (su.first_name, su.last_name) username = default_converter(username)[:auth_user_username_limit] su.username = username su.save() except Exception,e: logger.error(str(('cannot find/delete screensaver_user_id', ssuid, e))) ssuid = 830 # for ruchir shahs old acct try: su = ScreensaverUser.objects.get(screensaver_user_id=ssuid) username = '******' % (su.first_name, su.last_name) username = default_converter(username)[:auth_user_username_limit] su.username = username su.save() except Exception,e: logger.error(str(('cannot find/delete screensaver_user_id', ssuid, e)))
# delete this inconsistent user: # 866 | 14 | 2007-05-24 00:00:00-04 | Ernebjerg | Morten | [email protected] | 617-432-6392 | | using wellmate | | | me44 | 70572885 | | | | | | | | me44 ssuid = 866 try: obj = ScreensaverUser.objects.get(screensaver_user_id=ssuid) obj.delete() except Exception, e: logger.error(str(('cannot find/delete screensaver_user_id', ssuid, e))) # remove the second jgq10 erroneous account ssuid = 3166 try: su = ScreensaverUser.objects.get(screensaver_user_id=ssuid) username = '******' % (su.first_name, su.last_name) username = default_converter(username)[:auth_user_username_limit] su.username = username su.save() except Exception, e: logger.error(str(('cannot find/delete screensaver_user_id', ssuid, e))) ssuid = 830 # for ruchir shahs old acct try: su = ScreensaverUser.objects.get(screensaver_user_id=ssuid) username = '******' % (su.first_name, su.last_name) username = default_converter(username)[:auth_user_username_limit] su.username = username su.save() except Exception, e: logger.error(str(('cannot find/delete screensaver_user_id', ssuid, e)))
def create_checklist_vocabularies(apps): # create a separate vocab file: checklist_item_vocab, add to api_init.csv # output vocabs into a vocabulary patch file vocab_file = os.path.join( PROJECT_ROOT, '..', 'db', 'static', 'api_init', 'vocabulary_checklists_data.csv') logger.info('write vocabularies to %s' % vocab_file) resource_uri = '/reports/api/v1/vocabulary/%s/%s/' with open(vocab_file, 'w') as _file: vocab_writer = csv.writer(_file) header = ['resource_uri', 'key', 'scope', 'ordinal', 'title', 'expire_interval_days', 'expire_notifiy_days', ''] vocab_writer.writerow(header) # ci_group_map = {} scope = 'checklistitem.group' default_order = ['mailing', 'forms', 'non-harvard', 'imaging', 'legacy'] for obj in (apps.get_model('db', 'ChecklistItem') .objects.all().distinct('checklist_item_group')): key = default_converter(obj.checklist_item_group) ordinal = 0 for i, x in enumerate(default_order): if x in obj.checklist_item_group.lower(): ordinal = i break title = obj.checklist_item_group row = [resource_uri % (scope, key), key, scope, ordinal, title, None, None] vocab_writer.writerow(row) # ci_group_map[obj.checklist_item_group] = key logger.info(str(('created', row))) _scope = 'checklistitem.%s.name' # ci_name_map = {} for obj in (apps.get_model('db', 'ChecklistItem') .objects.all().distinct('item_name')): key = default_converter(obj.item_name) scope = _scope % default_converter(obj.checklist_item_group) # NOTE: fore user_checklist_item overload of vocabularies: if key in ('current_rnai_user_agreement_active', 'current_small_molecule_user_agreement_active'): expire_interval_days = 720 expire_notifiy_days = 60 else: expire_interval_days = None expire_notifiy_days = None title = obj.item_name ordinal = obj.order_statistic row = [resource_uri % (scope, key), key, scope, ordinal, title, expire_interval_days, expire_notifiy_days ] vocab_writer.writerow(row) # ci_name_map[obj.item_name] = key logger.info(str(('created', row))) scope = 'checklistitem.status' status_values = [ { 'key': 'not_completed', 'title': 'Not Completed', 'ordinal': 0 }, { 'key': 'activated', 'title': 'Activated', 'ordinal': 1 }, { 'key': 'deactivated', 'title': 'Deactivated', 'ordinal': 2 }, { 'key': 'na', 'title': 'N/A', 'ordinal': 3 }, { 'key': 'completed', 'title': 'Completed', 'ordinal': 4 }, ] for _dict in status_values: _dict['scope'] = scope row = [resource_uri % (_dict['scope'], _dict['key']), _dict['key'], _dict['scope'], _dict['ordinal'], _dict['title'], None, None] vocab_writer.writerow(row) logger.info(str(('created', row))) api_init_actions_file = os.path.join( lims.settings.PROJECT_ROOT, '..', 'db', 'static', 'api_init', 'api_init_actions.csv') logger.info('write %s entry to %s' % (vocab_file, api_init_actions_file)) with open(api_init_actions_file, 'a+') as _file: new_row = ['patch', 'vocabulary', os.path.basename(vocab_file)] reader = csv.reader(_file) found = False for row in reader: if row == new_row: found = True break if not found: logger.info('write api_init row: %s' % new_row) writer = csv.writer(_file) writer.writerow(new_row) else: logger.info('api_init entry for row already created: %r' % row) logger.info('vocabulary creation done')