def sync_score_row(legacy_row):
legacy_score_code = legacy_row[2]
legacy_scorer = legacy_row[3]
experiment = get_experiment(legacy_row[0], tile_to_well(legacy_row[1]))
score_code = get_score_code(legacy_score_code)
scorer = get_user(legacy_scorer)
timestamp = get_timestamp_from_ymd(legacy_row[4], legacy_row[5])
if not timestamp:
raise CommandError(
'ERROR: score of experiment {}, well {} '
'could not be converted to a proper datetime'.format(
legacy_row[0], legacy_row[1]))
new_score = ManualScore(experiment=experiment,
score_code=score_code,
scorer=scorer,
timestamp=timestamp)
return update_or_save_object(command,
new_score,
recorded_scores,
fields_to_compare,
alternate_pk={
'experiment': new_score.experiment,
'score_code': new_score.score_code,
'scorer': new_score.scorer,
'timestamp': new_score.timestamp
})
def sync_score_row(legacy_row):
legacy_score_code = legacy_row[2]
legacy_scorer = legacy_row[3]
experiment = get_experiment(legacy_row[0], tile_to_well(legacy_row[1]))
score_code = get_score_code(legacy_score_code)
scorer = get_user(legacy_scorer)
timestamp = get_timestamp_from_ymd(legacy_row[4], legacy_row[5])
if not timestamp:
raise CommandError(
'ERROR: score of experiment {}, well {} '
'could not be converted to a proper datetime'
.format(legacy_row[0], legacy_row[1]))
new_score = ManualScore(
experiment=experiment,
score_code=score_code,
scorer=scorer,
timestamp=timestamp)
return update_or_save_object(
command, new_score, recorded_scores, fields_to_compare,
alternate_pk={'experiment': new_score.experiment,
'score_code': new_score.score_code,
'scorer': new_score.scorer,
'timestamp': new_score.timestamp})
def sync_score_code_row(legacy_row):
new_score_code = ManualScoreCode(
id=legacy_row[0],
legacy_description=legacy_row[1].decode('utf8'))
return update_or_save_object(
command, new_score_code, recorded_score_codes, fields_to_compare)
def sync_score_row(legacy_row):
legacy_score_code = legacy_row[2]
legacy_scorer = legacy_row[3]
gene = legacy_row[9]
screen = legacy_row[10]
# Skip some scores entirely (see criteria above)
skip_entirely = ('alejandro', 'katy')
skip_ENH = ('eliana', 'lara')
if ((legacy_scorer == 'Julie' and legacy_score_code != -2)
or (legacy_scorer in skip_entirely)
or (screen == 'ENH' and legacy_scorer in skip_ENH)):
command.stderr.write(
'FYI: Skipping a {} {} score of {} by {}\n'.format(
gene, screen, legacy_score_code, legacy_scorer))
return True
# Convert some scorers to hueyling (see criteria above)
if legacy_scorer == 'Julie' or legacy_scorer == 'expPeople':
command.stderr.write(
'FYI: Converting a {} score by {} to hueyling\n'.format(
legacy_score_code, legacy_scorer))
legacy_scorer = 'hueyling'
# Convert some score codes (see criteria above)
if legacy_score_code == -6:
command.stderr.write('FYI: Converting score from -6 to -7\n')
legacy_score_code = -7
# Following raise exceptions if improperly formatted or not found
experiment = get_experiment(legacy_row[0], tile_to_well(legacy_row[1]))
score_code = get_score_code(legacy_score_code)
scorer = get_user(legacy_scorer)
timestamp = get_timestamp(legacy_row[5], legacy_row[6], legacy_row[7],
legacy_row[8], legacy_row[4])
if not timestamp:
raise CommandError(
'ERROR: score of {} could not be converted to a proper '
'datetime'.format(experiment))
new_score = ManualScore(experiment=experiment,
score_code=score_code,
scorer=scorer,
timestamp=timestamp)
return update_or_save_object(command,
new_score,
recorded_scores,
fields_to_compare,
alternate_pk={
'experiment': new_score.experiment,
'score_code': new_score.score_code,
'scorer': new_score.scorer,
'timestamp': new_score.timestamp
})
def sync_score_row(legacy_row):
legacy_score_code = legacy_row[2]
legacy_scorer = legacy_row[3]
gene = legacy_row[9]
screen = legacy_row[10]
# Skip some scores entirely (see criteria above)
skip_entirely = ('alejandro', 'katy')
skip_ENH = ('eliana', 'lara')
if ((legacy_scorer == 'Julie' and legacy_score_code != -2) or
(legacy_scorer in skip_entirely) or
(screen == 'ENH' and legacy_scorer in skip_ENH)):
command.stderr.write(
'FYI: Skipping a {} {} score of {} by {}\n'
.format(gene, screen, legacy_score_code, legacy_scorer))
return True
# Convert some scorers to hueyling (see criteria above)
if legacy_scorer == 'Julie' or legacy_scorer == 'expPeople':
command.stderr.write(
'FYI: Converting a {} score by {} to hueyling\n'
.format(legacy_score_code, legacy_scorer))
legacy_scorer = 'hueyling'
# Convert some score codes (see criteria above)
if legacy_score_code == -6:
command.stderr.write('FYI: Converting score from -6 to -7\n')
legacy_score_code = -7
# Following raise exceptions if improperly formatted or not found
experiment = get_experiment(legacy_row[0], tile_to_well(legacy_row[1]))
score_code = get_score_code(legacy_score_code)
scorer = get_user(legacy_scorer)
timestamp = get_timestamp(legacy_row[5], legacy_row[6], legacy_row[7],
legacy_row[8], legacy_row[4])
if not timestamp:
raise CommandError(
'ERROR: score of {} could not be converted to a proper '
'datetime'.format(experiment))
new_score = ManualScore(
experiment=experiment,
score_code=score_code,
scorer=scorer,
timestamp=timestamp)
return update_or_save_object(
command, new_score, recorded_scores, fields_to_compare,
alternate_pk={'experiment': new_score.experiment,
'score_code': new_score.score_code,
'scorer': new_score.scorer,
'timestamp': new_score.timestamp})
def sync_experiment_row(legacy_row):
experiment_plate_id = legacy_row[0]
worm_strain = get_worm_strain(legacy_row[1], legacy_row[2])
legacy_library_plate_name = legacy_row[3]
temperature = legacy_row[4]
date = legacy_row[5]
is_junk = legacy_row[6]
comment = legacy_row[7]
all_match = True
if experiment_plate_id < 40000:
screen_stage = 1
else:
screen_stage = 2
new_plate = ExperimentPlate(id=experiment_plate_id,
screen_stage=screen_stage,
temperature=temperature,
date=date,
comment=comment)
all_match &= update_or_save_object(command, new_plate, recorded_plates,
plate_fields_to_compare)
experiment_plate = get_experiment_plate(experiment_plate_id)
for well in get_well_list():
new_well = Experiment(
id=generate_experiment_id(experiment_plate_id, well),
plate=experiment_plate,
well=well,
worm_strain=worm_strain,
library_stock=get_library_stock(legacy_library_plate_name,
well),
is_junk=is_junk)
all_match &= update_or_save_object(command, new_well,
recorded_wells,
well_fields_to_compare)
return all_match
def sync_experiment_row(legacy_row):
experiment_plate_id = legacy_row[0]
worm_strain = get_worm_strain(legacy_row[1], legacy_row[2])
legacy_library_plate_name = legacy_row[3]
temperature = legacy_row[4]
date = legacy_row[5]
is_junk = legacy_row[6]
comment = legacy_row[7]
all_match = True
if experiment_plate_id < 40000:
screen_stage = 1
else:
screen_stage = 2
new_plate = ExperimentPlate(
id=experiment_plate_id,
screen_stage=screen_stage,
temperature=temperature,
date=date,
comment=comment)
all_match &= update_or_save_object(
command, new_plate, recorded_plates, plate_fields_to_compare)
experiment_plate = get_experiment_plate(experiment_plate_id)
for well in get_well_list():
new_well = Experiment(
id=generate_experiment_id(experiment_plate_id, well),
plate=experiment_plate, well=well,
worm_strain=worm_strain,
library_stock=get_library_stock(
legacy_library_plate_name, well),
is_junk=is_junk)
all_match &= update_or_save_object(
command, new_well, recorded_wells,
well_fields_to_compare)
return all_match
def sync_secondary_L4440_row(legacy_row):
plate_name = legacy_row[0]
well = get_three_character_well(legacy_row[1])
new_well = LibraryStock(
id=generate_library_stock_name(plate_name, well),
plate=get_library_plate(plate_name), well=well,
parent_stock=None, intended_clone=get_clone('L4440'))
return update_or_save_object(command, new_well, recorded_wells,
fields_to_compare)
def sync_library_plate_row(legacy_row, screen_stage=None,
number_of_wells=96):
if len(legacy_row) > 1:
plate_name = generate_ahringer_384_plate_name(legacy_row[0],
legacy_row[1])
else:
plate_name = generate_library_plate_name(legacy_row[0])
new_plate = LibraryPlate(id=plate_name, screen_stage=screen_stage,
number_of_wells=number_of_wells)
return update_or_save_object(command, new_plate, recorded_plates,
fields_to_compare)
def sync_primary_row(legacy_row):
plate_name = legacy_row[0]
well = get_three_character_well(legacy_row[1])
clone_name = legacy_row[2]
parent_plate_name = legacy_row[4]
if re.match('sjj', clone_name):
parent_chromosome = legacy_row[3]
parent_plate_name = generate_ahringer_384_plate_name(
parent_chromosome, parent_plate_name)
parent_well_improper = legacy_row[5]
if re.match('mv', clone_name):
clone_name = generate_vidal_clone_name(parent_plate_name,
parent_well_improper)
intended_clone = get_clone(clone_name)
if re.match('L4440', clone_name):
parent_stock = None
else:
parent_well_proper = get_three_character_well(parent_well_improper)
parent_stock = get_library_stock(parent_plate_name,
parent_well_proper)
# Confirm that this intended clone matches parent's clone
if parent_stock.intended_clone != intended_clone:
raise CommandError('Clone {} does not match parent\n'
.format(clone_name))
new_well = LibraryStock(
id=generate_library_stock_name(plate_name, well),
plate=get_library_plate(plate_name), well=well,
parent_stock=parent_stock,
intended_clone=intended_clone)
return update_or_save_object(command, new_well, recorded_wells,
fields_to_compare)
def sync_source_row(legacy_row):
plate_name = legacy_row[0]
clone_name = legacy_row[3]
if re.match('sjj', clone_name):
chromosome = legacy_row[2]
plate_name = generate_ahringer_384_plate_name(
chromosome, plate_name)
well_improper = legacy_row[1]
well_proper = get_three_character_well(well_improper)
if re.match('mv', clone_name):
clone_name = generate_vidal_clone_name(plate_name,
well_improper)
new_well = LibraryStock(
id=generate_library_stock_name(plate_name, well_proper),
plate=get_library_plate(plate_name), well=well_proper,
parent_stock=None, intended_clone=get_clone(clone_name))
return update_or_save_object(command, new_well, recorded_wells,
fields_to_compare)
def sync_eliana_row(legacy_row):
plate_name = legacy_row[0]
well = get_three_character_well(legacy_row[1])
if legacy_row[2]:
try:
parent_stock = get_library_stock(legacy_row[2], legacy_row[3])
intended_clone = parent_stock.intended_clone
except ObjectDoesNotExist:
parent_stock = None
intended_clone = None
else:
parent_stock = None
intended_clone = None
new_well = LibraryStock(
id=generate_library_stock_name(plate_name, well),
plate=get_library_plate(plate_name), well=well,
parent_stock=parent_stock,
intended_clone=intended_clone)
return update_or_save_object(command, new_well, recorded_wells,
fields_to_compare)
def sync_secondary_row(legacy_row):
plate_name = legacy_row[0]
well = get_three_character_well(legacy_row[1])
clone_name = legacy_row[2]
definite_parent_plate_name = legacy_row[3]
definite_parent_well = legacy_row[4]
if definite_parent_well:
definite_parent_well = get_three_character_well(
definite_parent_well)
likely_parent_plate_name = legacy_row[5]
likely_parent_well = legacy_row[6]
if likely_parent_well:
likely_parent_well = get_three_character_well(likely_parent_well)
likely_parent_clone_name = legacy_row[7]
if (definite_parent_plate_name and likely_parent_plate_name and
definite_parent_plate_name != likely_parent_plate_name):
raise CommandError(
'ERROR: definite and likely parent plates disagree '
'for {} {}\n'.format(plate_name, well))
if (definite_parent_well and likely_parent_well and
definite_parent_well != likely_parent_well):
raise CommandError(
'ERROR: definite and likely parent wells disagree '
'for {} {}\n'.format(plate_name, well))
try:
if definite_parent_plate_name and definite_parent_well:
parent_stock = get_library_stock(definite_parent_plate_name,
definite_parent_well)
else:
parent_stock = get_library_stock(likely_parent_plate_name,
likely_parent_well)
intended_clone = parent_stock.intended_clone
except ObjectDoesNotExist:
command.stderr.write(
'WARNING for LibraryStock {} {}: parent not '
'found in LibraryStock\n'.format(plate_name, well))
parent_stock = None
intended_clone = None
if clone_name and (clone_name != likely_parent_clone_name):
command.stderr.write(
'WARNING for LibraryStock {} {}: clone recorded '
'in CherryPickRNAiPlate is inconsistent with '
'CherryPickTemplate source/destination records\n'
.format(plate_name, well))
if re.match('sjj', clone_name):
try:
recorded_clone = get_clone(clone_name)
if recorded_clone != intended_clone:
command.stderr.write(
'WARNING for LibraryStock {} {}: clone recorded '
'in CherryPickRNAiPlate does not match its '
'parent\'s clone\n'.format(plate_name, well))
except ObjectDoesNotExist:
command.stderr.write(
'WARNING for LibraryStock {} {}: clone recorded in '
'CherryPickRNAiPlate not found at all in RNAiPlate\n'
.format(plate_name, well))
new_well = LibraryStock(
id=generate_library_stock_name(plate_name, well),
plate=get_library_plate(plate_name), well=well,
parent_stock=parent_stock,
intended_clone=intended_clone)
return update_or_save_object(command, new_well, recorded_wells,
fields_to_compare)
def sync_score_code_row(legacy_row):
new_score_code = ManualScoreCode(
id=legacy_row[0], legacy_description=legacy_row[1].decode('utf8'))
return update_or_save_object(command, new_score_code,
recorded_score_codes, fields_to_compare)
def sync_score_row(legacy_row):
# Build the object using the minimimum fields
count_adult = legacy_row[8]
count_larva = legacy_row[9]
if count_adult == -1:
count_adult = None
if count_larva == -1:
count_larva = None
machine_call = legacy_row[15]
if machine_call:
selected_for_scoring = True
else:
selected_for_scoring = False
new_score = DevstarScore(
experiment=get_experiment(legacy_row[0], legacy_row[1]),
area_adult=legacy_row[5],
area_larva=legacy_row[6],
area_embryo=legacy_row[7],
count_adult=count_adult,
count_larva=count_larva,
is_bacteria_present=legacy_row[16],
selected_for_scoring=selected_for_scoring,
gi_score_larva_per_adult=legacy_row[17],
gi_score_survival=legacy_row[18]
)
# Clean the object to populate the fields derived from other fields
new_score.clean()
###############
# SANITY CHECKS
###############
errors = []
new_allele = new_score.experiment.worm_strain.allele
if (new_allele != legacy_row[2]):
# Deal with case of legacy database using zc310 instead of zu310
if (legacy_row[2] == 'zc310' and
new_allele == 'zu310'):
pass
# Deal with some case sensitivity issues in legacy database
# (e.g. see experiment 32405, where the allele is capitalized).
# Can't do lower() in all cases because "N2" should always be
# capitalized.
elif new_allele == legacy_row[2].lower():
pass
else:
errors.append('allele mismatch')
# Deal with case of some experiments having the wrong RNAiPlateID
# in the legacy database's RawDataWithScore table. This field is
# redundant with the RawData table, and RawData is more trustworthy;
# however it is still worthwhile to perform this check in order
# to find the mismatches, and to confirm manually that each one
# makes sense.
new_lp = new_score.experiment.library_stock.plate_id
legacy_lp = legacy_row[4].replace('_', '-').replace('zc310', 'zu310')
if (legacy_lp != new_lp and
new_score.experiment.plate_id not in (461, 8345) and
('vidal-' not in new_lp or
legacy_lp != new_lp.split('vidal-')[1])):
errors.append('RNAi plate mismatch: {} {}')
if new_score.count_embryo != legacy_row[10]:
errors.append('embryo count mismatch')
if (new_score.embryo_per_adult and legacy_row[8] and
legacy_row[8] != -1 and
int(new_score.embryo_per_adult) != legacy_row[11]):
errors.append('embryo per adult mismatch')
if (new_score.larva_per_adult and legacy_row[9] and
legacy_row[9] != -1 and
int(new_score.larva_per_adult) != legacy_row[12]):
errors.append('larva per adult mismatch')
if (new_score.survival and not compare_floats_for_equality(
new_score.survival, legacy_row[13]) and
legacy_row[13] != 0):
errors.append('invalid survival')
if (new_score.lethality and not compare_floats_for_equality(
new_score.lethality, legacy_row[14]) and
legacy_row[13] != 0):
errors.append('invalid lethality')
if errors:
raise CommandError(
'DevstarScore for {}:{} had these errors: {}'
.format(legacy_row[0], legacy_row[1], errors))
return update_or_save_object(
command, new_score, recorded_scores, fields_to_compare,
alternate_pk={'experiment': new_score.experiment})
def sync_clone_row(legacy_row):
new_clone = Clone(id=legacy_row[0])
return update_or_save_object(command, new_clone, recorded_clones,
fields_to_compare)
def sync_clone_row_vidal(legacy_row):
vidal_clone_name = generate_vidal_clone_name(legacy_row[0],
legacy_row[1])
new_clone = Clone(id=vidal_clone_name)
return update_or_save_object(command, new_clone, recorded_clones,
fields_to_compare)
def sync_score_row(legacy_row):
# Build the object using the minimimum fields
count_adult = legacy_row[8]
count_larva = legacy_row[9]
if count_adult == -1:
count_adult = None
if count_larva == -1:
count_larva = None
machine_call = legacy_row[15]
if machine_call:
selected_for_scoring = True
else:
selected_for_scoring = False
new_score = DevstarScore(experiment=get_experiment(
legacy_row[0], legacy_row[1]),
area_adult=legacy_row[5],
area_larva=legacy_row[6],
area_embryo=legacy_row[7],
count_adult=count_adult,
count_larva=count_larva,
is_bacteria_present=legacy_row[16],
selected_for_scoring=selected_for_scoring,
gi_score_larva_per_adult=legacy_row[17],
gi_score_survival=legacy_row[18])
# Clean the object to populate the fields derived from other fields
new_score.clean()
###############
# SANITY CHECKS
###############
errors = []
new_allele = new_score.experiment.worm_strain.allele
if (new_allele != legacy_row[2]):
# Deal with case of legacy database using zc310 instead of zu310
if (legacy_row[2] == 'zc310' and new_allele == 'zu310'):
pass
# Deal with some case sensitivity issues in legacy database
# (e.g. see experiment 32405, where the allele is capitalized).
# Can't do lower() in all cases because "N2" should always be
# capitalized.
elif new_allele == legacy_row[2].lower():
pass
else:
errors.append('allele mismatch')
# Deal with case of some experiments having the wrong RNAiPlateID
# in the legacy database's RawDataWithScore table. This field is
# redundant with the RawData table, and RawData is more trustworthy;
# however it is still worthwhile to perform this check in order
# to find the mismatches, and to confirm manually that each one
# makes sense.
new_lp = new_score.experiment.library_stock.plate_id
legacy_lp = legacy_row[4].replace('_', '-').replace('zc310', 'zu310')
if (legacy_lp != new_lp
and new_score.experiment.plate_id not in (461, 8345)
and ('vidal-' not in new_lp
or legacy_lp != new_lp.split('vidal-')[1])):
errors.append('RNAi plate mismatch: {} {}')
if new_score.count_embryo != legacy_row[10]:
errors.append('embryo count mismatch')
if (new_score.embryo_per_adult and legacy_row[8]
and legacy_row[8] != -1
and int(new_score.embryo_per_adult) != legacy_row[11]):
errors.append('embryo per adult mismatch')
if (new_score.larva_per_adult and legacy_row[9] and legacy_row[9] != -1
and int(new_score.larva_per_adult) != legacy_row[12]):
errors.append('larva per adult mismatch')
if (new_score.survival and not compare_floats_for_equality(
new_score.survival, legacy_row[13]) and legacy_row[13] != 0):
errors.append('invalid survival')
if (new_score.lethality and not compare_floats_for_equality(
new_score.lethality, legacy_row[14]) and legacy_row[13] != 0):
errors.append('invalid lethality')
if errors:
raise CommandError(
'DevstarScore for {}:{} had these errors: {}'.format(
legacy_row[0], legacy_row[1], errors))
return update_or_save_object(
command,
new_score,
recorded_scores,
fields_to_compare,
alternate_pk={'experiment': new_score.experiment})
