def xls_write_workbook(file, data, request=None, image_keys=None,
title_function=None, list_brackets=None):
'''
***WARNING*** xlsx files load fully into memory on display - if there are
>~ 2000 images, this will cause performance issues on the client.***
@param sheet_rows iterable of dicts, one per row
'''
if not isinstance(data, dict):
raise BadRequest(
'unknown data for generic xls serialization: %r' % type(data))
wb = xlsxwriter.Workbook(file, {'constant_memory': True})
logger.info('xls_write_workbook for data: %r', data.keys())
for key, sheet_rows in data.items():
logger.info('type sheet_rows: %r', type(sheet_rows))
if isinstance(sheet_rows, (dict, OrderedDict)):
sheet_name = default_converter(key)
logger.info('writing sheet %r...', sheet_name)
sheet = wb.add_worksheet(sheet_name)
for i, row in enumerate(csvutils.dict_to_rows(sheet_rows)):
sheet.write_row(i,0,row)
elif isinstance(sheet_rows, basestring):
sheet_name = default_converter(key)
logger.info('writing single string sheet %r...', sheet_name)
sheet = wb.add_worksheet(sheet_name)
sheet.write_string(0,0,sheet_rows)
else:
sheet_name = default_converter(key)
logger.info('writing sheets for base name %r...', sheet_name)
write_rows_to_sheet(wb, sheet_rows, sheet_name, request=request,
image_keys=image_keys, title_function=title_function,
list_brackets=list_brackets)
wb.close()
def generic_xls_write_workbook(file, data):
'''
Writes a dict of iterables to a workbook,
where key=sheet name, value=iterable that is ready to write (e.g. a list)
'''
wb = xlsxwriter.Workbook(file, {'constant_memory': True})
if isinstance(data, dict):
logger.info('generic_xls_write_workbook for data: %r', data.keys())
for key, sheet_rows in data.items():
sheet_name = default_converter(key)
logger.info('writing sheet %r...', sheet_name)
sheet = wb.add_worksheet(sheet_name)
if isinstance(sheet_rows, dict):
for i, row in enumerate(csvutils.dict_to_rows(sheet_rows)):
sheet.write_row(i,0,row)
elif isinstance(sheet_rows, basestring):
sheet.write_string(0,0,sheet_rows)
else:
generic_write_rows_to_sheet(sheet_rows, sheet)
else:
raise BadRequest(
'unknown data for generic xls serialization: %r' % type(data))
logger.info('save to file; %r', file.name)
wb.close()
def generic_xls_write_workbook(file, data):
'''
Writes a dict of iterables to a workbook,
where key=sheet name, value=iterable that is ready to write (e.g. a list)
'''
wb = xlsxwriter.Workbook(file, {'constant_memory': True})
if isinstance(data, dict):
logger.info('generic_xls_write_workbook for data: %r', data.keys())
for key, sheet_rows in data.items():
sheet_name = default_converter(key)
logger.info('writing sheet %r...', sheet_name)
sheet = wb.add_worksheet(sheet_name)
if isinstance(sheet_rows, dict):
for i, row in enumerate(csvutils.dict_to_rows(sheet_rows)):
sheet.write_row(i, 0, row)
elif isinstance(sheet_rows, basestring):
sheet.write_string(0, 0, sheet_rows)
else:
generic_write_rows_to_sheet(sheet_rows, sheet)
else:
raise BadRequest('unknown data for generic xls serialization: %r' %
type(data))
logger.info('save to file; %r', file.name)
wb.close()
def create_attached_file_type_vocab(apps):
vocab_file = os.path.join(
lims.settings.PROJECT_ROOT, '..',
'db', 'static', 'api_init', 'vocabulary_attachedfiletype_data.csv')
logger.info('write vocabularies to %s' % vocab_file)
resource_uri = '/reports/api/v1/vocabulary/%s/%s/'
with open(vocab_file, 'w') as _file:
vocab_writer = csv.writer(_file)
header = ['resource_uri', 'key', 'scope', 'ordinal', 'title', 'is_retired']
vocab_writer.writerow(header)
_scope = 'attachedfiletype.%s'
for i, obj in enumerate(apps.get_model('db', 'AttachedFileType')
.objects.all().order_by('value')):
key = obj.value.lower()
key = key.replace('nsrb','') # For Jen Smith
key = default_converter(obj.value)
scope = _scope % obj.for_entity_type
title = obj.value
ordinal = i
is_retired = key in [
'marcus_application','miare_document',
'nerce_screener_supplies_list']
row = [resource_uri % (scope, key), key, scope, ordinal, title, is_retired]
vocab_writer.writerow(row)
logger.info(str(('created', row)))
(apps.get_model('db', 'AttachedFile').objects
.filter(attached_file_type=obj).update(type=key))
# For Jen Smith - 20160720
# /reports/api/v1/vocabulary/attachedfiletype.screen/material_transfer_agreement material_transfer_agreement attachedfiletype.screen 21 Material Transfer Agreement
# /reports/api/v1/vocabulary/attachedfiletype.screen/other other attachedfiletype.screen 22 Other
# /reports/api/v1/vocabulary/attachedfiletype.user/other other attachedfiletype.user 22 Other
# row = [resource_uri % (scope, key), key, scope, ordinal, title, is_retired]
# vocab_writer.writerow(row)
api_init_actions_file = os.path.join(
lims.settings.PROJECT_ROOT, '..',
'db', 'static', 'api_init', 'api_init_actions.csv')
logger.info('write %s entry to %s' % (vocab_file, api_init_actions_file))
with open(api_init_actions_file, 'a+') as _file:
new_row = ['patch', 'vocabulary', os.path.basename(vocab_file)]
reader = csv.reader(_file)
found = False
for row in reader:
if row == new_row:
found = True
break
if not found:
writer = csv.writer(_file)
writer.writerow(new_row)
else:
logger.info('api_init entry for row already created: %r' % row)
logger.info('done')
def xls_write_workbook(file,
data,
request=None,
image_keys=None,
title_function=None,
list_brackets=None):
'''
***WARNING*** xlsx files load fully into memory on display - if there are
>~ 2000 images, this will cause performance issues on the client.***
@param sheet_rows iterable of dicts, one per row
'''
if not isinstance(data, dict):
raise BadRequest('unknown data for generic xls serialization: %r' %
type(data))
wb = xlsxwriter.Workbook(file, {'constant_memory': True})
logger.info('xls_write_workbook for data: %r', data.keys())
for key, sheet_rows in data.items():
logger.info('type sheet_rows: %r', type(sheet_rows))
if isinstance(sheet_rows, (dict, OrderedDict)):
sheet_name = default_converter(key)
logger.info('writing sheet %r...', sheet_name)
sheet = wb.add_worksheet(sheet_name)
for i, row in enumerate(csvutils.dict_to_rows(sheet_rows)):
sheet.write_row(i, 0, row)
elif isinstance(sheet_rows, basestring):
sheet_name = default_converter(key)
logger.info('writing single string sheet %r...', sheet_name)
sheet = wb.add_worksheet(sheet_name)
sheet.write_string(0, 0, sheet_rows)
else:
sheet_name = default_converter(key)
logger.info('writing sheets for base name %r...', sheet_name)
write_rows_to_sheet(wb,
sheet_rows,
sheet_name,
request=request,
image_keys=image_keys,
title_function=title_function,
list_brackets=list_brackets)
wb.close()
def result_value_generator(result_values):
logger.info('Write the result values sheet')
header_row = []
header_row.extend(RESULT_VALUE_FIELD_MAP.keys())
# TODO: allow column titles to be optional
header_row.extend([fields[key].get('title', key) for key in data_columns])
header_row.extend(other_columns)
row_count = 0
for result_value in result_values:
row_count += 1
row = []
row.extend(result_value['well_id'].split(':'))
if ( result_value.has_key('assay_well_control_type')
and result_value['assay_well_control_type'] ):
control_type = default_converter(result_value['assay_well_control_type'])
# note: "empty", "experimental", "buffer" are values that can be
# found in this column, due to legacy data entry, but they are
# not valid
if control_type in control_type_mapping:
row.append(control_type_mapping[control_type])
else:
row.append(None)
else:
row.append(None)
excluded_cols = []
if result_value.has_key('exclude') and result_value['exclude']:
temp = result_value['exclude']
if hasattr(temp, 'split'):
temp = temp.split(LIST_DELIMITER_SQL_ARRAY)
logger.debug('excluded data_columns: find %r, in %r', temp, data_columns)
for data_column_name in temp:
excluded_cols.append(get_column_letter(
len(RESULT_VALUE_FIELD_MAP)+1
+data_columns.index(data_column_name)))
excluded_cols = sorted(excluded_cols)
row.append(','.join(excluded_cols))
for j,key in enumerate(data_columns):
if result_value.has_key(key):
row.append(result_value[key])
else:
row.append(None)
# append the non-result value columns to the end of the row
for j,key in enumerate(other_columns):
if result_value.has_key(key):
row.append(result_value[key])
if row_count % 10000 == 0:
logger.info('wrote %d rows', row_count)
yield OrderedDict(zip(header_row,row))
def parse_columns(columns_sheet):
logger.info('parsing columns sheet: %r', columns_sheet.name)
columns = data_column_generator(sheet_cols(columns_sheet))
parsed_cols = OrderedDict()
errors = {}
for i,column in enumerate(columns):
parsed_col = {
'is_derived': False,
'is_follow_up_data': False,
'ordinal': i
}
logger.debug('parsing column: %r', column['data_worksheet_column'])
parsed_cols[column['data_worksheet_column']] = parsed_col
for key,val in column.items():
if key == 'is_follow_up_data':
parsed_col[key] = ( val and val.lower() == 'follow up')
elif key == 'data_type':
val = default_converter(val)
# handle validation errors in the api
if val not in DATA_TYPES:
key = '%s:%s' % (column['data_worksheet_column'],'data_type')
errors[key] = 'val: %r must be one of %r' % (val,DATA_TYPES)
parsed_col[key] = val
elif key == 'assay_readout_type':
parsed_col[key] = default_converter(val)
else:
if key == 'how_derived':
parsed_col['is_derived'] = (
val is not None and val.strip() is not '' )
parsed_col[key] = val
if parsed_col.get('decimal_places', None):
try:
key = '%s:%s' % (column['data_worksheet_column'],'data_type')
column['decimal_places'] = parse_val(
column['decimal_places'],key,'integer')
except ValidationError, e:
errors.update(e.errors)
logger.debug('parsed_col: %r', parsed_col)
def create_serviceactivity_vocab(vocab_writer, attr, scope, query):
resource_uri = '/reports/api/v1/vocabulary/%s/%s/'
logger.info('create simple vocab: %s, %s', attr, scope)
vocabs = []
for ordinal, attr_value in (enumerate(
query.values_list(attr, flat=True).distinct(attr).order_by(attr))):
if not attr_value: continue
key = default_converter(attr_value)
title = attr_value
_resource_uri = resource_uri % (scope, key)
vocabs.append([_resource_uri, key, scope, ordinal, title])
for row in vocabs:
title = row[4]
key = row[1]
# query.filter(**{ '%s__exact' % attr: title }).update(**{ attr: key })
vocab_writer.writerow(row)
logger.info('updated vocab: %r' % row)
def migrate_screen_status(apps, schema_editor):
'''
Migrate the screen_status_item table to the "status" field of the screen
object and create the needed ApiLog entries to record the history
NOTE: manual migration 0002 must be run first:
- adds an "id" field to screen_status_item
- copies the latest status to the screen.status field
'''
# Create a history log for all of the status's for each screen,
# and store the _current/latest_ status on the new screen.status field
count = 0
ScreenStatusItem = apps.get_model('db', 'ScreenStatusItem')
for screen in (apps.get_model(
'db', 'Screen').objects.all().order_by('facility_id')):
logger.info('process screen: %s', screen.facility_id)
if screen.status:
# Clean up the vocabulary used in the status_item table
# NOTE: migration 0003 shows a newer way of generating the vocabs
# - this is ok for these
screen.status = default_converter(screen.status)
screen.save()
# now scan the screen_status_items to recreate logs
prev_item = None
for status in (ScreenStatusItem.objects.filter(
screen=screen).order_by('status_date')):
diffs = {}
if prev_item:
diffs['status'] = [prev_item.status, status.status]
else:
diffs['status'] = [None, status.status]
log = make_log(apps,
status.status_date,
'screen',
screen.facility_id,
diffs,
username='******')
logger.debug('created log: %d: %r', count, log)
prev_item = status
count = count + 1
logger.info('updated: %d screen status entries', count)
def create_serviceactivity_vocab(vocab_writer, attr, scope, query):
resource_uri = '/reports/api/v1/vocabulary/%s/%s/'
logger.info('create simple vocab: %s, %s', attr,scope)
vocabs = []
for ordinal, attr_value in (enumerate(
query.values_list(attr, flat=True)
.distinct(attr).order_by(attr))):
if not attr_value: continue
key = default_converter(attr_value)
title = attr_value
_resource_uri = resource_uri % (scope, key)
vocabs.append([_resource_uri, key, scope, ordinal, title])
for row in vocabs:
title = row[4]
key = row[1]
# query.filter(**{ '%s__exact' % attr: title }).update(**{ attr: key })
vocab_writer.writerow(row)
logger.info('updated vocab: %r' % row)
def data_column_field_mapper(fields):
mapped_keys = []
for key in fields:
key = key.lower()
mapped_key = None
if key in DATA_COLUMN_FIELD_MAP:
mapped_key = DATA_COLUMN_FIELD_MAP[key]
else:
for k in DATA_COLUMN_FIELD_MAP.values():
if default_converter(key) == k:
mapped_key = k
break
if not mapped_key:
raise ParseError(
key=key,
msg=('key %r is not in the recognized datacolumn fields: %r'
% (key,DATA_COLUMN_FIELD_MAP.keys())))
mapped_keys.append(mapped_key)
return mapped_keys
def migrate_screen_status(apps,schema_editor):
'''
Migrate the screen_status_item table to the "status" field of the screen
object and create the needed ApiLog entries to record the history
NOTE: manual migration 0002 must be run first:
- adds an "id" field to screen_status_item
- copies the latest status to the screen.status field
'''
# Create a history log for all of the status's for each screen,
# and store the _current/latest_ status on the new screen.status field
count=0
ScreenStatusItem = apps.get_model('db','ScreenStatusItem')
for screen in (
apps.get_model('db','Screen').objects.all().order_by('facility_id')):
logger.info('process screen: %s', screen.facility_id)
if screen.status:
# Clean up the vocabulary used in the status_item table
# NOTE: migration 0003 shows a newer way of generating the vocabs
# - this is ok for these
screen.status = default_converter(screen.status)
screen.save()
# now scan the screen_status_items to recreate logs
prev_item = None
for status in ( ScreenStatusItem.objects.filter(screen=screen)
.order_by('status_date')):
diffs = {}
if prev_item:
diffs['status'] = [prev_item.status, status.status]
else:
diffs['status'] = [None, status.status]
log = make_log(
apps,status.status_date, 'screen', screen.facility_id, diffs, username='******')
logger.debug('created log: %d: %r', count, log)
prev_item = status
count = count + 1
logger.info('updated: %d screen status entries', count)
def create_user_checklist_items(apps, schema_editor):
# prerequisites:
# - convert checklist_item / checklist_item_event entries into into
# checklistitem.* vocabularies (migration 0003)
# - create the user_checklist_item table (0002)
ChecklistItem = apps.get_model('db', 'ChecklistItem')
UserChecklistItem = apps.get_model('db', 'UserChecklistItem')
ci_group_map = {}
for obj in ChecklistItem.objects.all().distinct('checklist_item_group'):
key = default_converter(obj.checklist_item_group)
ci_group_map[obj.checklist_item_group] = key
ci_name_map = {}
for obj in ChecklistItem.objects.all().distinct('item_name'):
key = default_converter(obj.item_name)
ci_name_map[obj.item_name] = key
# create entries in the user_checklist_item table
# note: status values are hard-coded to correspond to the vocabulary
# keys (created in migration 0002)
sql_keys = [
'suid', 'cigroup', 'ciname', 'su_username', 'admin_username',
'admin_suid', 'admin_upid', 'date_performed', 'date_created', 'status',
'is_notified'
]
sql = '''
select
screening_room_user_id,
ci.checklist_item_group,
ci.item_name,
su.username su_username,
admin.username admin_username,
admin.screensaver_user_id admin_suid,
up.id admin_upid,
cie.date_performed,
cie.date_created,
case when cie.is_not_applicable then 'n_a'
when ci.is_expirable and cie.date_performed is not null then
case when cie.is_expiration then 'deactivated' else 'activated' end
when cie.date_performed is not null then 'completed'
else 'not_completed'
end as status,
(
select 1 from screening_room_user sru
where sru.last_notified_smua_checklist_item_event_id = cie.checklist_item_event_id
UNION
select 1 from screening_room_user sru
where sru.last_notified_rnaiua_checklist_item_event_id = cie.checklist_item_event_id
) as is_notified
from checklist_item ci
join checklist_item_event cie using(checklist_item_id)
join screensaver_user su on screening_room_user_id=su.screensaver_user_id
join screensaver_user admin on cie.created_by_id=admin.screensaver_user_id
left join reports_userprofile up on up.id=admin.user_id
order by screening_room_user_id, checklist_item_group, item_name, cie.date_performed asc;
'''
connection = schema_editor.connection
cursor = connection.cursor()
log_ref_resource_name = 'userchecklistitem'
_dict = None
log = None
uci_hash = {}
unique_log_keys = set()
try:
cursor.execute(sql)
i = 0
for row in cursor:
_dict = dict(zip(sql_keys, row))
key = '/'.join(
[str(_dict['suid']), _dict['cigroup'], _dict['ciname']])
previous_dict = uci_hash.get(key)
logger.debug('previous_dict: %s:%s' % (key, previous_dict))
date_time = pytz.timezone('US/Eastern').localize(
_dict['date_created'])
if date_time.date() != _dict['date_performed']:
# only use the less accurate date_performed date if that date
# is not equal to the date_created date
date_time = pytz.timezone('US/Eastern').localize(
datetime.datetime.combine(_dict['date_performed'],
datetime.datetime.min.time()))
if previous_dict:
uci = previous_dict['obj']
uci.admin_user_id = int(_dict['admin_suid'])
uci.status = _dict['status']
uci.previous_status = previous_dict['status']
if (previous_dict['is_notified']):
# notified date will be this event - 60 days (smua/rnaiua)
uci.status_notified_date = (_dict['date_performed'] -
datetime.timedelta(days=60))
uci.status_date = _dict['date_performed']
logger.debug(
'saving, dict: %s, prev_dict: %s, status date %s, status_notified: %s',
_dict, previous_dict, uci.status_date,
uci.status_notified_date)
uci.save()
logger.debug('update uci: %s,%s,%s,%s', uci.status,
uci.status_date, uci.previous_status,
uci.status_notified_date)
else:
uci_hash[key] = _dict
logger.debug(
str(('create user checklist item', _dict,
_dict['date_performed'].isoformat())))
uci = UserChecklistItem.objects.create(
screensaver_user_id=int(_dict['suid']),
admin_user_id=int(_dict['admin_suid']),
item_group=ci_group_map[_dict['cigroup']],
item_name=ci_name_map[_dict['ciname']],
status=_dict['status'],
status_date=_dict['date_performed'])
uci.save()
_dict['obj'] = uci
logger.debug('created uci: %s,%s,%s', uci.status,
uci.status_date)
i += 1
# create the apilog for this item
log = ApiLog()
log.ref_resource_name = log_ref_resource_name
log.key = '/'.join(
[_dict['su_username'], uci.item_group, uci.item_name])
log.username = _dict['admin_username']
log.user_id = _dict['admin_upid']
log.date_time = date_time
log.api_action = 'PATCH'
log.uri = '/'.join([log.ref_resource_name, log.key])
log.comment = 'status=%s' % _dict['status']
# is the key (date_time, actually) unique?
full_key = '/'.join(
[log.ref_resource_name, log.key,
str(log.date_time)])
while full_key in unique_log_keys:
# add a second to make it unique; because date performed is a date,
logger.info(str(('time collision for: ', full_key)))
log.date_time = log.date_time + datetime.timedelta(0, 1)
full_key = '/'.join(
[log.ref_resource_name, log.key,
str(log.date_time)])
unique_log_keys.add(full_key)
if previous_dict:
diff_keys = ['status']
diffs = {}
logger.debug(str(('found previous_dict', previous_dict)))
diff_keys.append('admin_username')
diffs['admin_username'] = [
previous_dict['admin_username'], _dict['admin_username']
]
diff_keys.append('status_date')
diffs['status_date'] = [
previous_dict['date_performed'].isoformat(),
_dict['date_performed'].isoformat()
]
diffs['status'] = [previous_dict['status'], _dict['status']]
diff_keys.append('previous_status')
diffs['previous_status'] = [None, previous_dict['status']]
log.diff_keys = json.dumps(diff_keys)
log.diffs = json.dumps(diffs)
logger.debug('create log: %s', log)
log.save()
log = None
if i % 1000 == 0:
logger.info(str(('created', i, 'logs')))
except Exception, e:
logger.exception('migration exc')
raise e
def create_checklist_vocabularies(apps):
# create a separate vocab file: checklist_item_vocab, add to api_init.csv
# output vocabs into a vocabulary patch file
vocab_file = os.path.join(PROJECT_ROOT, '..', 'db', 'static', 'api_init',
'vocabulary_checklists_data.csv')
logger.info('write vocabularies to %s' % vocab_file)
resource_uri = '/reports/api/v1/vocabulary/%s/%s/'
with open(vocab_file, 'w') as _file:
vocab_writer = csv.writer(_file)
header = [
'resource_uri', 'key', 'scope', 'ordinal', 'title',
'expire_interval_days', 'expire_notifiy_days', ''
]
vocab_writer.writerow(header)
# ci_group_map = {}
scope = 'checklistitem.group'
default_order = [
'mailing', 'forms', 'non-harvard', 'imaging', 'legacy'
]
for obj in (apps.get_model(
'db',
'ChecklistItem').objects.all().distinct('checklist_item_group')
):
key = default_converter(obj.checklist_item_group)
ordinal = 0
for i, x in enumerate(default_order):
if x in obj.checklist_item_group.lower():
ordinal = i
break
title = obj.checklist_item_group
row = [
resource_uri % (scope, key), key, scope, ordinal, title, None,
None
]
vocab_writer.writerow(row)
# ci_group_map[obj.checklist_item_group] = key
logger.info(str(('created', row)))
_scope = 'checklistitem.%s.name'
# ci_name_map = {}
for obj in (apps.get_model(
'db', 'ChecklistItem').objects.all().distinct('item_name')):
key = default_converter(obj.item_name)
scope = _scope % default_converter(obj.checklist_item_group)
# NOTE: fore user_checklist_item overload of vocabularies:
if key in ('current_rnai_user_agreement_active',
'current_small_molecule_user_agreement_active'):
expire_interval_days = 720
expire_notifiy_days = 60
else:
expire_interval_days = None
expire_notifiy_days = None
title = obj.item_name
ordinal = obj.order_statistic
row = [
resource_uri % (scope, key), key, scope, ordinal, title,
expire_interval_days, expire_notifiy_days
]
vocab_writer.writerow(row)
# ci_name_map[obj.item_name] = key
logger.info(str(('created', row)))
scope = 'checklistitem.status'
status_values = [
{
'key': 'not_completed',
'title': 'Not Completed',
'ordinal': 0
},
{
'key': 'activated',
'title': 'Activated',
'ordinal': 1
},
{
'key': 'deactivated',
'title': 'Deactivated',
'ordinal': 2
},
{
'key': 'na',
'title': 'N/A',
'ordinal': 3
},
{
'key': 'completed',
'title': 'Completed',
'ordinal': 4
},
]
for _dict in status_values:
_dict['scope'] = scope
row = [
resource_uri % (_dict['scope'], _dict['key']), _dict['key'],
_dict['scope'], _dict['ordinal'], _dict['title'], None, None
]
vocab_writer.writerow(row)
logger.info(str(('created', row)))
api_init_actions_file = os.path.join(lims.settings.PROJECT_ROOT, '..',
'db', 'static', 'api_init',
'api_init_actions.csv')
logger.info('write %s entry to %s' % (vocab_file, api_init_actions_file))
with open(api_init_actions_file, 'a+') as _file:
new_row = ['patch', 'vocabulary', os.path.basename(vocab_file)]
reader = csv.reader(_file)
found = False
for row in reader:
if row == new_row:
found = True
break
if not found:
logger.info('write api_init row: %s' % new_row)
writer = csv.writer(_file)
writer.writerow(new_row)
else:
logger.info('api_init entry for row already created: %r' % row)
logger.info('vocabulary creation done')
def create_attached_file_type_vocab(apps):
vocab_file = os.path.join(lims.settings.PROJECT_ROOT, '..', 'db', 'static',
'api_init',
'vocabulary_attachedfiletype_data.csv')
logger.info('write vocabularies to %s' % vocab_file)
resource_uri = '/reports/api/v1/vocabulary/%s/%s/'
with open(vocab_file, 'w') as _file:
vocab_writer = csv.writer(_file)
header = [
'resource_uri', 'key', 'scope', 'ordinal', 'title', 'is_retired'
]
vocab_writer.writerow(header)
_scope = 'attachedfiletype.%s'
for i, obj in enumerate(
apps.get_model(
'db', 'AttachedFileType').objects.all().order_by('value')):
key = obj.value.lower()
key = key.replace('nsrb', '') # For Jen Smith
key = default_converter(obj.value)
scope = _scope % obj.for_entity_type
title = obj.value
ordinal = i
is_retired = key in [
'marcus_application', 'miare_document',
'nerce_screener_supplies_list'
]
row = [
resource_uri % (scope, key), key, scope, ordinal, title,
is_retired
]
vocab_writer.writerow(row)
logger.info(str(('created', row)))
(apps.get_model('db', 'AttachedFile').objects.filter(
attached_file_type=obj).update(type=key))
# For Jen Smith - 20160720
# /reports/api/v1/vocabulary/attachedfiletype.screen/material_transfer_agreement material_transfer_agreement attachedfiletype.screen 21 Material Transfer Agreement
# /reports/api/v1/vocabulary/attachedfiletype.screen/other other attachedfiletype.screen 22 Other
# /reports/api/v1/vocabulary/attachedfiletype.user/other other attachedfiletype.user 22 Other
# row = [resource_uri % (scope, key), key, scope, ordinal, title, is_retired]
# vocab_writer.writerow(row)
api_init_actions_file = os.path.join(lims.settings.PROJECT_ROOT, '..',
'db', 'static', 'api_init',
'api_init_actions.csv')
logger.info('write %s entry to %s' % (vocab_file, api_init_actions_file))
with open(api_init_actions_file, 'a+') as _file:
new_row = ['patch', 'vocabulary', os.path.basename(vocab_file)]
reader = csv.reader(_file)
found = False
for row in reader:
if row == new_row:
found = True
break
if not found:
writer = csv.writer(_file)
writer.writerow(new_row)
else:
logger.info('api_init entry for row already created: %r' % row)
logger.info('done')
def create_lab_affiliation_vocab(apps):
# populate the title field, change the name field to a key
replace_phrases = [['harvard medical school', 'hms'],
['harvard university', 'harvard'],
['European Molecular Biology Laboratory', 'embl'],
['[embl]', ''],
['Dana Farber Cancer Institute', 'dfci'],
['University of California', 'uc'],
['University of Massachusetts', 'umass'],
['Institute of Chemistry and Cell Biology', 'iccb'],
['Beth Israel Deaconess Medical Center', 'bidmc'],
['Tufts University', 'tufts'],
['University of California', 'UC'],
['University of Massachusetts', 'umass'], ['[NYU]', ''],
['the', ''], ['of', ''], ['in', ''],
["women's", 'womens'], ["children's", 'childrens']]
replace_phrases = [[re.compile(r'\b%s\b' % x, re.IGNORECASE), y]
for [x, y] in replace_phrases]
vocab_file = os.path.join(lims.settings.PROJECT_ROOT, '..', 'db', 'static',
'api_init',
'vocabulary_lab_affiliations_data.csv')
logger.info('write vocabularies to %s' % vocab_file)
resource_uri = 'vocabulary/%s/%s/'
with open(vocab_file, 'a+') as _file:
header = [
'resource_uri', 'key', 'scope', 'ordinal', 'title', 'comment'
]
reader = csv.reader(_file)
vocab_writer = csv.writer(_file)
defined_vocabs = {}
try:
header = reader.next()
logger.info('read header: %s', header)
for row in reader:
defined_vocabs[row[4]] = dict(zip(header, row))
except StopIteration as e:
logger.info('no entries in %s, writing a new file', vocab_file)
vocab_writer.writerow(header)
scope = 'labaffiliation.category'
default_order = [
'hms', 'hms_affiliated_hospital', 'hsph', 'harvard_fas',
'broad_icg', 'other'
]
for i, la in enumerate(
apps.get_model('db', 'LabAffiliation').objects.all().distinct(
'affiliation_category')):
if la.affiliation_category not in defined_vocabs:
key = default_converter(la.affiliation_category)
if key in default_order:
ordinal = default_order.index(key)
else:
ordinal = i + len(default_order)
title = la.affiliation_category
row = [
resource_uri % (scope, key), key, scope, ordinal, title,
la.affiliation_category
]
vocab_writer.writerow(row)
defined_vocabs[la.affiliation_category] = dict(zip(
header, row))
logger.debug('created %s', row)
else:
logger.info('vocabulary already exists: %s - %s',
la.affiliation_category,
defined_vocabs[la.affiliation_category])
_scope = 'labaffiliation.category.%s'
count_updated = 0
for i, la in enumerate(
apps.get_model('db', 'LabAffiliation').objects.all().order_by(
'affiliation_category', 'affiliation_name')):
if la.affiliation_name not in defined_vocabs:
name = la.affiliation_name.lower()
for replacer, replacement in replace_phrases:
logger.info('replacer: %s, replacement: %s, name: %s',
str(replacer), replacement, name)
name = replacer.sub(replacement.lower(), name)
logger.info('new name: %s', name)
title = la.affiliation_name
key = default_converter(name)
scope = _scope % default_converter(la.affiliation_category)
ordinal = len(defined_vocabs) + 1
row = [
resource_uri % (scope, key), key, scope, ordinal, title,
la.affiliation_category
]
defined_vocabs[la.affiliation_name] = dict(zip(header, row))
vocab_writer.writerow(row)
logger.debug('created row: %s', row)
else:
logger.info('vocabulary already exists: %s - %s',
la.affiliation_name,
defined_vocabs[la.affiliation_name])
# now set the screensaveruser field
ScreensaverUser = apps.get_model('db', 'ScreensaverUser')
if la.labhead_set.exists():
for lh in la.labhead_set.all():
su = ScreensaverUser.objects.get(
screensaver_user_id=lh.screensaver_user_id)
new_value = defined_vocabs[la.affiliation_name]['key']
logger.debug('updating user %s, lab_affiliation: %s',
su.username, new_value)
su.lab_head_affiliation = new_value
su.save()
count_updated += 1
logger.info(
'labaffiliation vocabulary creation done, updated: %s users, %s vocabs',
count_updated, len(defined_vocabs))
def create_plate_activity_logs(apps, schema_editor):
logger.info('create plate activity logs')
Activity = apps.get_model('db', 'Activity')
cols = OrderedDict({
'activity_id': 'a.activity_id',
'username': '******',
'screensaver_user_id': 'screensaver_user_id',
'date_of_activity': 'date_of_activity',
'comments': 'a.comments',
'plate_number': 'plate_number',
'copy_name': 'copy.name',
'library_short_name': 'library.short_name',
})
colkeys = cols.keys()
_cols = ', '.join([ '%s as %s' % (value,key) for key, value in cols.items() ])
sql = (
'select ' + _cols +
''' from activity a join screensaver_user on(performed_by_id=screensaver_user_id)
join plate on (activity_id=plate.plated_activity_id)
join copy using(copy_id)
join library using(library_id); '''
)
connection = schema_editor.connection
cursor = connection.cursor()
cursor.execute(sql)
_list = cursor.fetchall()
if len(_list) == 0:
raise Exception('No plate plated_activities found with sql: %r' % sql)
for i,_data in enumerate(_list):
_activity = dict(zip(colkeys, _data))
log = ApiLog()
log.ref_resource_name = librarycopyplate_resource_name
log.key = '/'.join([
_activity['library_short_name'],_activity['copy_name'],
str(int(_activity['plate_number'])).zfill(5)])
log.uri = '/'.join([base_uri,log.ref_resource_name,log.key])
log.comment = _activity['comments']
log.date_time = create_log_time(_activity['date_of_activity'])
log.username = _activity['username']
log.user_id = _activity['screensaver_user_id']
if "'available'" in log.comment.lower():
log.diffs = { 'status': ['not_specied','available']}
elif "'not available'" in log.comment.lower():
log.diffs = { 'status': ['not_specied','not_available']}
else:
raise Exception('unknown plate.plated_activity comment: %r', _activity)
log.save()
if i % 1000 == 0:
logger.info('processed %d plate plated activity logs', i)
logger.info('processed %d plate plated activity logs', i)
sql = (
'select ' + _cols +
''' from activity a join screensaver_user on(performed_by_id=screensaver_user_id)
join plate on (activity_id=plate.retired_activity_id)
join copy using(copy_id)
join library using(library_id); '''
)
cursor.execute(sql)
_list = cursor.fetchall()
status_change_pattern = re.compile(r".*from '([^\']+)'.*to '([^\']+)'.*")
if len(_list) == 0:
raise Exception('No plate retired_activities found with sql: %r' % sql)
status_terms_recognized = set()
for i,_data in enumerate(_list):
_activity = dict(zip(colkeys, _data))
log = ApiLog()
log.ref_resource_name = librarycopyplate_resource_name
log.key = '/'.join([
_activity['library_short_name'],_activity['copy_name'],
str(int(_activity['plate_number'])).zfill(5)])
log.uri = '/'.join([base_uri,log.ref_resource_name,log.key])
log.comment = _activity['comments']
log.date_time = create_log_time(_activity['date_of_activity'])
log.username = _activity['username']
log.user_id = _activity['screensaver_user_id']
match = status_change_pattern.match(log.comment)
if not match:
raise Exception('unknown plate.retired_activity comment: %r', _activity)
log.diffs = {'status': [
default_converter(match.group(1)),
default_converter(match.group(2))]}
log.save()
status_terms_recognized.add(default_converter(match.group(1)))
status_terms_recognized.add(default_converter(match.group(2)))
if i % 1000 == 0:
logger.info('processed %d plate retired activity logs', i)
logger.info('processed %d plate retired activity logs', i)
logger.info('status terms recognized: %r', status_terms_recognized)
def create_lab_affiliation_vocab(apps):
# populate the title field, change the name field to a key
replace_phrases = [
['harvard medical school', 'hms'],
['harvard university', 'harvard'],
['European Molecular Biology Laboratory', 'embl'],
['[embl]',''],
['Dana Farber Cancer Institute', 'dfci'],
['University of California', 'uc'],
['University of Massachusetts', 'umass'],
['Institute of Chemistry and Cell Biology', 'iccb'],
['Beth Israel Deaconess Medical Center', 'bidmc'],
['Tufts University', 'tufts'],
['University of California', 'UC'],
['University of Massachusetts', 'umass'],
['[NYU]', ''],
['the',''],
['of',''],
['in',''],
["women's", 'womens'],
["children's", 'childrens']
]
replace_phrases = [[re.compile(r'\b%s\b' % x, re.IGNORECASE),y]
for [x,y] in replace_phrases ]
vocab_file = os.path.join(
lims.settings.PROJECT_ROOT, '..',
'db', 'static', 'api_init', 'vocabulary_lab_affiliations_data.csv')
logger.info('write vocabularies to %s' % vocab_file)
resource_uri = 'vocabulary/%s/%s/'
with open(vocab_file, 'a+') as _file:
header = ['resource_uri', 'key', 'scope', 'ordinal', 'title', 'comment']
reader = csv.reader(_file)
vocab_writer = csv.writer(_file)
defined_vocabs = {}
try:
header = reader.next()
logger.info('read header: %s', header)
for row in reader:
defined_vocabs[row[4]] = dict(zip(header,row))
except StopIteration as e:
logger.info('no entries in %s, writing a new file',vocab_file)
vocab_writer.writerow(header)
scope = 'labaffiliation.category'
default_order = ['hms','hms_affiliated_hospital','hsph',
'harvard_fas','broad_icg','other']
for i,la in enumerate(apps.get_model('db', 'LabAffiliation')
.objects.all().distinct('affiliation_category')):
if la.affiliation_category not in defined_vocabs:
key = default_converter(la.affiliation_category)
if key in default_order:
ordinal = default_order.index(key)
else:
ordinal = i + len(default_order)
title = la.affiliation_category
row = [resource_uri % (scope, key), key, scope, ordinal, title,
la.affiliation_category ]
vocab_writer.writerow(row)
defined_vocabs[la.affiliation_category] = dict(zip(header,row))
logger.debug('created %s', row)
else:
logger.info('vocabulary already exists: %s - %s',
la.affiliation_category, defined_vocabs[la.affiliation_category])
_scope = 'labaffiliation.category.%s'
count_updated = 0
for i,la in enumerate(apps.get_model('db', 'LabAffiliation')
.objects.all()
.order_by('affiliation_category','affiliation_name')):
if la.affiliation_name not in defined_vocabs:
name = la.affiliation_name.lower()
for replacer,replacement in replace_phrases:
logger.info('replacer: %s, replacement: %s, name: %s', str(replacer),replacement,name)
name = replacer.sub(replacement.lower(),name)
logger.info('new name: %s', name)
title = la.affiliation_name
key = default_converter(name)
scope = _scope % default_converter(la.affiliation_category)
ordinal = len(defined_vocabs) + 1
row = [resource_uri % (scope, key), key, scope, ordinal, title,
la.affiliation_category ]
defined_vocabs[la.affiliation_name] = dict(zip(header,row))
vocab_writer.writerow(row)
logger.debug('created row: %s', row)
else:
logger.info('vocabulary already exists: %s - %s',
la.affiliation_name, defined_vocabs[la.affiliation_name])
# now set the screensaveruser field
ScreensaverUser = apps.get_model('db','ScreensaverUser')
if la.labhead_set.exists():
for lh in la.labhead_set.all():
su = ScreensaverUser.objects.get(screensaver_user_id=lh.screensaver_user_id)
new_value = defined_vocabs[la.affiliation_name]['key']
logger.debug('updating user %s, lab_affiliation: %s', su.username,new_value )
su.lab_head_affiliation = new_value;
su.save()
count_updated += 1
logger.info('labaffiliation vocabulary creation done, updated: %s users, %s vocabs',
count_updated, len(defined_vocabs))
def get_xls_response(
data, output_filename,request=None,image_keys=None,
title_function=None, list_brackets=None):
'''
Create an xlsx file that will be streamed through the StreamingHttpResponse.
- if length exceeds MAX_ROWS_PER_XLS_FILE, create multiple files and zip them.
- TODO: when using xlsx, can simply add extra sheets to the file.
@param output_filename - for naming temp files
FIXME: wrap cursor with cursorgenerator; pass in the image columns as arg
FIXME: rework this using the generic_xlsx_response as a template:
- this method is used for all xlsx serialization at this time, except
for in testing, and in ScreenResultSerializer - 20160419.
'''
if not isinstance(data, dict):
raise BadRequest(
'unknown data for xls serialization: %r, must be a dict of '
'sheet_row entries' % type(data))
# create a temp dir
# with TemporaryFile() as f:
temp_dir = os.path.join(
settings.TEMP_FILE_DIR, str(time.clock()).replace('.', '_'))
os.mkdir(temp_dir)
try:
# Create an new Excel file and add a worksheet.
filename = '%s.xlsx' % (output_filename)
temp_file = os.path.join(temp_dir, filename)
file_names_to_zip = [temp_file]
if DEBUG_STREAMING: logger.info('temp file: %r', temp_file)
workbook = xlsxwriter.Workbook(temp_file, {'constant_memory': True})
for key, sheet_rows in data.items():
logger.info('type sheet_rows: %r', type(sheet_rows))
if isinstance(sheet_rows, (dict, OrderedDict)):
sheet_name = default_converter(key)
logger.info('writing sheet %r...', sheet_name)
sheet = workbook.add_worksheet(sheet_name)
for i, row in enumerate(csvutils.dict_to_rows(sheet_rows)):
sheet.write_row(i,0,row)
elif isinstance(sheet_rows, basestring):
sheet_name = default_converter(key)
logger.info('writing single string sheet %r...', sheet_name)
sheet = workbook.add_worksheet(sheet_name)
sheet.write_string(0,0,sheet_rows)
else:
sheet_name = default_converter(key)
logger.info('writing sheets for base name %r...', sheet_name)
max_rows_per_sheet = 2**20
sheet = workbook.add_worksheet(sheet_name)
filerow = 0
sheets = 1
for row,values in enumerate(sheet_rows):
if filerow == 0:
for i,(key,val) in enumerate(values.items()):
title = key
if title_function:
title = title_function(key)
sheet.write_string(filerow,i,title)
filerow += 1
for i, (key,val) in enumerate(values.items()):
val = csvutils.csv_convert(
val, delimiter=LIST_DELIMITER_XLS,
list_brackets=list_brackets)
if val is not None:
if len(val) > 32767:
logger.error('warn, row too long, %d, key: %r, len: %d',
row,key,len(val) )
if image_keys and key in image_keys:
max_rows_per_sheet = MAX_IMAGE_ROWS_PER_XLS_FILE
if not request:
raise Exception(
'must specify the request parameter for image export')
# hack to speed things up:
if ( key == 'structure_image' and
'library_well_type' in values and
values['library_well_type'].lower() == 'empty' ):
continue
write_xls_image(sheet, filerow, i, val, request)
else:
sheet.write_string(filerow,i,val)
filerow += 1
if row % 10000 == 0:
logger.info('wrote %d rows to temp file', row)
if filerow > max_rows_per_sheet:
workbook.close()
logger.info('wrote file: %r', temp_file)
# Create an new Excel file and add a worksheet.
filename = '%s_%s.xlsx' % (output_filename, filerow)
temp_file = os.path.join(temp_dir, filename)
workbook = xlsxwriter.Workbook(temp_file, {'constant_memory': True})
sheet = workbook.add_worksheet(sheet_name)
file_names_to_zip.append(temp_file)
filerow = 0
workbook.close()
logger.info('wrote file: %r', temp_file)
content_type = '%s; charset=utf-8' % XLSX_MIMETYPE
if len(file_names_to_zip) >1:
# create a temp zip file
content_type='application/zip; charset=utf-8'
temp_file = os.path.join('/tmp',str(time.clock()))
logger.info('temp ZIP file: %r', temp_file)
with ZipFile(temp_file, 'w') as zip_file:
for _file in file_names_to_zip:
zip_file.write(_file, os.path.basename(_file))
logger.info('wrote file %r', temp_file)
filename = '%s.zip' % output_filename
_file = file(temp_file)
logger.info('download tmp file: %r, %r',temp_file,_file)
wrapper = FileWrapper(_file)
response = StreamingHttpResponse(
wrapper, content_type=content_type)
response['Content-Length'] = os.path.getsize(temp_file)
response['Content-Disposition'] = \
'attachment; filename=%s' % filename
return response
except Exception, e:
logger.exception('xls streaming error')
raise e
def create_user_checklist_items(apps, schema_editor):
# prerequisites:
# - convert checklist_item / checklist_item_event entries into into
# checklistitem.* vocabularies (migration 0003)
# - create the user_checklist_item table (0002)
ChecklistItem = apps.get_model('db','ChecklistItem')
UserChecklistItem = apps.get_model('db','UserChecklistItem')
ci_group_map = {}
for obj in ChecklistItem.objects.all().distinct('checklist_item_group'):
key = default_converter(obj.checklist_item_group)
ci_group_map[obj.checklist_item_group] = key
ci_name_map = {}
for obj in ChecklistItem.objects.all().distinct('item_name'):
key = default_converter(obj.item_name)
ci_name_map[obj.item_name] = key
# create entries in the user_checklist_item table
# note: status values are hard-coded to correspond to the vocabulary
# keys (created in migration 0002)
sql_keys = [
'suid','cigroup','ciname',
'su_username','admin_username','admin_suid','admin_upid',
'date_performed', 'date_created','status','is_notified'
]
sql = '''
select
screening_room_user_id,
ci.checklist_item_group,
ci.item_name,
su.username su_username,
admin.username admin_username,
admin.screensaver_user_id admin_suid,
up.id admin_upid,
cie.date_performed,
cie.date_created,
case when cie.is_not_applicable then 'n_a'
when ci.is_expirable and cie.date_performed is not null then
case when cie.is_expiration then 'deactivated' else 'activated' end
when cie.date_performed is not null then 'completed'
else 'not_completed'
end as status,
(
select 1 from screening_room_user sru
where sru.last_notified_smua_checklist_item_event_id = cie.checklist_item_event_id
UNION
select 1 from screening_room_user sru
where sru.last_notified_rnaiua_checklist_item_event_id = cie.checklist_item_event_id
) as is_notified
from checklist_item ci
join checklist_item_event cie using(checklist_item_id)
join screensaver_user su on screening_room_user_id=su.screensaver_user_id
join screensaver_user admin on cie.created_by_id=admin.screensaver_user_id
left join reports_userprofile up on up.id=admin.user_id
order by screening_room_user_id, checklist_item_group, item_name, cie.date_performed asc;
'''
connection = schema_editor.connection
cursor = connection.cursor()
log_ref_resource_name = 'userchecklistitem'
_dict = None
log = None
uci_hash = {}
unique_log_keys = set()
try:
cursor.execute(sql)
i = 0
for row in cursor:
_dict = dict(zip(sql_keys,row))
key = '/'.join([str(_dict['suid']),_dict['cigroup'],_dict['ciname']])
previous_dict = uci_hash.get(key)
logger.debug('previous_dict: %s:%s' % (key,previous_dict))
date_time = pytz.timezone('US/Eastern').localize(_dict['date_created'])
if date_time.date() != _dict['date_performed']:
# only use the less accurate date_performed date if that date
# is not equal to the date_created date
date_time = pytz.timezone('US/Eastern').localize(
datetime.datetime.combine(
_dict['date_performed'],
datetime.datetime.min.time()))
if previous_dict:
uci = previous_dict['obj']
uci.admin_user_id = int(_dict['admin_suid'])
uci.status = _dict['status']
uci.previous_status = previous_dict['status']
if(previous_dict['is_notified']):
# notified date will be this event - 60 days (smua/rnaiua)
uci.status_notified_date = (
_dict['date_performed'] - datetime.timedelta(days=60))
uci.status_date = _dict['date_performed']
logger.debug('saving, dict: %s, prev_dict: %s, status date %s, status_notified: %s',
_dict, previous_dict, uci.status_date, uci.status_notified_date)
uci.save()
logger.debug('update uci: %s,%s,%s,%s',
uci.status,uci.status_date,uci.previous_status,uci.status_notified_date)
else:
uci_hash[key] = _dict
logger.debug(str(('create user checklist item', _dict,
_dict['date_performed'].isoformat())))
uci = UserChecklistItem.objects.create(
screensaver_user_id = int(_dict['suid']),
admin_user_id = int(_dict['admin_suid']),
item_group = ci_group_map[_dict['cigroup']],
item_name = ci_name_map[_dict['ciname']],
status = _dict['status'],
status_date = _dict['date_performed'])
uci.save()
_dict['obj'] = uci
logger.debug('created uci: %s,%s,%s', uci.status, uci.status_date)
i += 1
# create the apilog for this item
log = ApiLog()
log.ref_resource_name = log_ref_resource_name
log.key = '/'.join([_dict['su_username'],uci.item_group,uci.item_name])
log.username = _dict['admin_username']
log.user_id = _dict['admin_upid']
log.date_time = date_time
log.api_action = 'PATCH'
log.uri = '/'.join([log.ref_resource_name,log.key])
log.comment = 'status=%s' % _dict['status']
# is the key (date_time, actually) unique?
full_key = '/'.join([log.ref_resource_name,log.key,str(log.date_time)])
while full_key in unique_log_keys:
# add a second to make it unique; because date performed is a date,
logger.info(str(('time collision for: ',full_key)))
log.date_time = log.date_time + datetime.timedelta(0,1)
full_key = '/'.join([log.ref_resource_name,log.key,str(log.date_time)])
unique_log_keys.add(full_key)
if previous_dict:
diff_keys = ['status']
diffs = {}
logger.debug(str(('found previous_dict', previous_dict)))
diff_keys.append('admin_username')
diffs['admin_username'] = [previous_dict['admin_username'], _dict['admin_username']]
diff_keys.append('status_date')
diffs['status_date'] = [
previous_dict['date_performed'].isoformat(),
_dict['date_performed'].isoformat()]
diffs['status'] = [previous_dict['status'],_dict['status']]
diff_keys.append('previous_status')
diffs['previous_status'] = [ None, previous_dict['status']]
log.diff_keys = json.dumps(diff_keys)
log.diffs = json.dumps(diffs)
logger.debug('create log: %s', log)
log.save()
log = None
if i%1000 == 0:
logger.info(str(('created', i, 'logs')))
except Exception, e:
logger.exception('migration exc')
raise e
# delete this inconsistent user:
# 866 | 14 | 2007-05-24 00:00:00-04 | Ernebjerg | Morten | [email protected] | 617-432-6392 | | using wellmate | | | me44 | 70572885 | | | | | | | | me44
ssuid = 866
try:
obj = ScreensaverUser.objects.get(screensaver_user_id=ssuid)
obj.delete()
except Exception,e:
logger.error(str(('cannot find/delete screensaver_user_id', ssuid, e)))
# remove the second jgq10 erroneous account
ssuid = 3166
try:
su = ScreensaverUser.objects.get(screensaver_user_id=ssuid)
username = '******' % (su.first_name, su.last_name)
username = default_converter(username)[:auth_user_username_limit]
su.username = username
su.save()
except Exception,e:
logger.error(str(('cannot find/delete screensaver_user_id', ssuid, e)))
ssuid = 830
# for ruchir shahs old acct
try:
su = ScreensaverUser.objects.get(screensaver_user_id=ssuid)
username = '******' % (su.first_name, su.last_name)
username = default_converter(username)[:auth_user_username_limit]
su.username = username
su.save()
except Exception,e:
logger.error(str(('cannot find/delete screensaver_user_id', ssuid, e)))
# delete this inconsistent user:
# 866 | 14 | 2007-05-24 00:00:00-04 | Ernebjerg | Morten | [email protected] | 617-432-6392 | | using wellmate | | | me44 | 70572885 | | | | | | | | me44
ssuid = 866
try:
obj = ScreensaverUser.objects.get(screensaver_user_id=ssuid)
obj.delete()
except Exception, e:
logger.error(str(('cannot find/delete screensaver_user_id', ssuid, e)))
# remove the second jgq10 erroneous account
ssuid = 3166
try:
su = ScreensaverUser.objects.get(screensaver_user_id=ssuid)
username = '******' % (su.first_name, su.last_name)
username = default_converter(username)[:auth_user_username_limit]
su.username = username
su.save()
except Exception, e:
logger.error(str(('cannot find/delete screensaver_user_id', ssuid, e)))
ssuid = 830
# for ruchir shahs old acct
try:
su = ScreensaverUser.objects.get(screensaver_user_id=ssuid)
username = '******' % (su.first_name, su.last_name)
username = default_converter(username)[:auth_user_username_limit]
su.username = username
su.save()
except Exception, e:
logger.error(str(('cannot find/delete screensaver_user_id', ssuid, e)))
def create_checklist_vocabularies(apps):
# create a separate vocab file: checklist_item_vocab, add to api_init.csv
# output vocabs into a vocabulary patch file
vocab_file = os.path.join(
PROJECT_ROOT, '..',
'db', 'static', 'api_init', 'vocabulary_checklists_data.csv')
logger.info('write vocabularies to %s' % vocab_file)
resource_uri = '/reports/api/v1/vocabulary/%s/%s/'
with open(vocab_file, 'w') as _file:
vocab_writer = csv.writer(_file)
header = ['resource_uri', 'key', 'scope', 'ordinal', 'title',
'expire_interval_days', 'expire_notifiy_days', '']
vocab_writer.writerow(header)
# ci_group_map = {}
scope = 'checklistitem.group'
default_order = ['mailing', 'forms', 'non-harvard', 'imaging', 'legacy']
for obj in (apps.get_model('db', 'ChecklistItem')
.objects.all().distinct('checklist_item_group')):
key = default_converter(obj.checklist_item_group)
ordinal = 0
for i, x in enumerate(default_order):
if x in obj.checklist_item_group.lower():
ordinal = i
break
title = obj.checklist_item_group
row = [resource_uri % (scope, key), key, scope, ordinal, title, None, None]
vocab_writer.writerow(row)
# ci_group_map[obj.checklist_item_group] = key
logger.info(str(('created', row)))
_scope = 'checklistitem.%s.name'
# ci_name_map = {}
for obj in (apps.get_model('db', 'ChecklistItem')
.objects.all().distinct('item_name')):
key = default_converter(obj.item_name)
scope = _scope % default_converter(obj.checklist_item_group)
# NOTE: fore user_checklist_item overload of vocabularies:
if key in ('current_rnai_user_agreement_active',
'current_small_molecule_user_agreement_active'):
expire_interval_days = 720
expire_notifiy_days = 60
else:
expire_interval_days = None
expire_notifiy_days = None
title = obj.item_name
ordinal = obj.order_statistic
row = [resource_uri % (scope, key), key, scope, ordinal, title,
expire_interval_days, expire_notifiy_days ]
vocab_writer.writerow(row)
# ci_name_map[obj.item_name] = key
logger.info(str(('created', row)))
scope = 'checklistitem.status'
status_values = [
{
'key': 'not_completed',
'title': 'Not Completed',
'ordinal': 0
},
{
'key': 'activated',
'title': 'Activated',
'ordinal': 1
},
{
'key': 'deactivated',
'title': 'Deactivated',
'ordinal': 2
},
{
'key': 'na',
'title': 'N/A',
'ordinal': 3
},
{
'key': 'completed',
'title': 'Completed',
'ordinal': 4
},
]
for _dict in status_values:
_dict['scope'] = scope
row = [resource_uri % (_dict['scope'], _dict['key']),
_dict['key'], _dict['scope'], _dict['ordinal'], _dict['title'], None, None]
vocab_writer.writerow(row)
logger.info(str(('created', row)))
api_init_actions_file = os.path.join(
lims.settings.PROJECT_ROOT, '..',
'db', 'static', 'api_init', 'api_init_actions.csv')
logger.info('write %s entry to %s' % (vocab_file, api_init_actions_file))
with open(api_init_actions_file, 'a+') as _file:
new_row = ['patch', 'vocabulary', os.path.basename(vocab_file)]
reader = csv.reader(_file)
found = False
for row in reader:
if row == new_row:
found = True
break
if not found:
logger.info('write api_init row: %s' % new_row)
writer = csv.writer(_file)
writer.writerow(new_row)
else:
logger.info('api_init entry for row already created: %r' % row)
logger.info('vocabulary creation done')
