def handle(self, **options):
require_db_write_acknowledgement()
return
if options['all']:
experiments = Experiment.objects.all()
else:
null_devstar_experiment_pks = (DevstarScore.objects.filter(
area_adult__isnull=True,
area_larva__isnull=True,
area_embryo__isnull=True).values_list('experiment', flat=True))
null_devstar_experiments = Experiment.objects.filter(
pk__in=null_devstar_experiment_pks)
all_devstar_experiment_pks = (
DevstarScore.objects.all().values_list('experiment',
flat=True))
no_devstar_experiments = Experiment.objects.exclude(
pk__in=all_devstar_experiment_pks)
experiments = sorted(chain(null_devstar_experiments,
no_devstar_experiments),
key=lambda instance: instance.id)
for experiment in experiments:
path = experiment.get_devstar_count_path()
if not os.path.isfile(path):
self.stderr.write('WARNING: Skipping experiment {} due to '
'DevStaR file not found'.format(experiment))
continue
counts = [None] * 6
with open(path, 'r') as f:
for line in f:
for i, pattern in enumerate(PATTERNS):
if line.startswith(pattern):
try:
counts[i] = int(line.split()[1])
except Exception:
raise CommandError('Error parsing line {} '
'in experiment {}'.format(
i, experiment))
for i, count in enumerate(counts):
if count is None:
self.stderr.write('WARNING: {} count is missing '
'for experiment {}'.format(
PATTERNS[i], experiment))
new_score = DevstarScore(
experiment=experiment,
is_bacteria_present=counts[0],
area_adult=counts[1],
area_larva=counts[2],
area_embryo=counts[3],
count_adult=counts[4],
count_larva=counts[5],
)
# If score already exists in database, check for match
try:
previous_score = DevstarScore.objects.get(
experiment=experiment)
if not previous_score.matches_raw_fields(new_score):
raise CommandError(
'The DevStaR txt output does '
'not match the existing database '
'entry for Experiment {}'.format(experiment))
except ObjectDoesNotExist:
self.stderr.write('DevStaR object not found for {}. '
'Attempting to save.'.format(experiment))
try:
new_score.full_clean()
except ValidationError:
raise CommandError(
'Cleaning DevStaR object {} '
'raised ValidationError'.format(new_score))
def sync_score_row(legacy_row):
# Build the object using the minimimum fields
count_adult = legacy_row[8]
count_larva = legacy_row[9]
if count_adult == -1:
count_adult = None
if count_larva == -1:
count_larva = None
machine_call = legacy_row[15]
if machine_call:
selected_for_scoring = True
else:
selected_for_scoring = False
new_score = DevstarScore(
experiment=get_experiment(legacy_row[0], legacy_row[1]),
area_adult=legacy_row[5],
area_larva=legacy_row[6],
area_embryo=legacy_row[7],
count_adult=count_adult,
count_larva=count_larva,
is_bacteria_present=legacy_row[16],
selected_for_scoring=selected_for_scoring,
gi_score_larva_per_adult=legacy_row[17],
gi_score_survival=legacy_row[18]
)
# Clean the object to populate the fields derived from other fields
new_score.clean()
###############
# SANITY CHECKS
###############
errors = []
new_allele = new_score.experiment.worm_strain.allele
if (new_allele != legacy_row[2]):
# Deal with case of legacy database using zc310 instead of zu310
if (legacy_row[2] == 'zc310' and
new_allele == 'zu310'):
pass
# Deal with some case sensitivity issues in legacy database
# (e.g. see experiment 32405, where the allele is capitalized).
# Can't do lower() in all cases because "N2" should always be
# capitalized.
elif new_allele == legacy_row[2].lower():
pass
else:
errors.append('allele mismatch')
# Deal with case of some experiments having the wrong RNAiPlateID
# in the legacy database's RawDataWithScore table. This field is
# redundant with the RawData table, and RawData is more trustworthy;
# however it is still worthwhile to perform this check in order
# to find the mismatches, and to confirm manually that each one
# makes sense.
new_lp = new_score.experiment.library_stock.plate_id
legacy_lp = legacy_row[4].replace('_', '-').replace('zc310', 'zu310')
if (legacy_lp != new_lp and
new_score.experiment.plate_id not in (461, 8345) and
('vidal-' not in new_lp or
legacy_lp != new_lp.split('vidal-')[1])):
errors.append('RNAi plate mismatch: {} {}')
if new_score.count_embryo != legacy_row[10]:
errors.append('embryo count mismatch')
if (new_score.embryo_per_adult and legacy_row[8] and
legacy_row[8] != -1 and
int(new_score.embryo_per_adult) != legacy_row[11]):
errors.append('embryo per adult mismatch')
if (new_score.larva_per_adult and legacy_row[9] and
legacy_row[9] != -1 and
int(new_score.larva_per_adult) != legacy_row[12]):
errors.append('larva per adult mismatch')
if (new_score.survival and not compare_floats_for_equality(
new_score.survival, legacy_row[13]) and
legacy_row[13] != 0):
errors.append('invalid survival')
if (new_score.lethality and not compare_floats_for_equality(
new_score.lethality, legacy_row[14]) and
legacy_row[13] != 0):
errors.append('invalid lethality')
if errors:
raise CommandError(
'DevstarScore for {}:{} had these errors: {}'
.format(legacy_row[0], legacy_row[1], errors))
return update_or_save_object(
command, new_score, recorded_scores, fields_to_compare,
alternate_pk={'experiment': new_score.experiment})
def handle(self, **options):
require_db_write_acknowledgement()
return
if options['all']:
experiments = Experiment.objects.all()
else:
null_devstar_experiment_pks = (
DevstarScore.objects.filter(
area_adult__isnull=True,
area_larva__isnull=True,
area_embryo__isnull=True)
.values_list('experiment', flat=True))
null_devstar_experiments = Experiment.objects.filter(
pk__in=null_devstar_experiment_pks)
all_devstar_experiment_pks = (
DevstarScore.objects.all()
.values_list('experiment', flat=True))
no_devstar_experiments = Experiment.objects.exclude(
pk__in=all_devstar_experiment_pks)
experiments = sorted(
chain(null_devstar_experiments, no_devstar_experiments),
key=lambda instance: instance.id)
for experiment in experiments:
path = experiment.get_devstar_count_path()
if not os.path.isfile(path):
self.stderr.write('WARNING: Skipping experiment {} due to '
'DevStaR file not found'
.format(experiment))
continue
counts = [None] * 6
with open(path, 'r') as f:
for line in f:
for i, pattern in enumerate(PATTERNS):
if line.startswith(pattern):
try:
counts[i] = int(line.split()[1])
except Exception:
raise CommandError('Error parsing line {} '
'in experiment {}'
.format(i, experiment))
for i, count in enumerate(counts):
if count is None:
self.stderr.write('WARNING: {} count is missing '
'for experiment {}'
.format(PATTERNS[i], experiment))
new_score = DevstarScore(
experiment=experiment,
is_bacteria_present=counts[0], area_adult=counts[1],
area_larva=counts[2], area_embryo=counts[3],
count_adult=counts[4], count_larva=counts[5],
)
# If score already exists in database, check for match
try:
previous_score = DevstarScore.objects.get(
experiment=experiment)
if not previous_score.matches_raw_fields(new_score):
raise CommandError('The DevStaR txt output does '
'not match the existing database '
'entry for Experiment {}'
.format(experiment))
except ObjectDoesNotExist:
self.stderr.write('DevStaR object not found for {}. '
'Attempting to save.'
.format(experiment))
try:
new_score.full_clean()
except ValidationError:
raise CommandError('Cleaning DevStaR object {} '
'raised ValidationError'
.format(new_score))
def sync_score_row(legacy_row):
# Build the object using the minimimum fields
count_adult = legacy_row[8]
count_larva = legacy_row[9]
if count_adult == -1:
count_adult = None
if count_larva == -1:
count_larva = None
machine_call = legacy_row[15]
if machine_call:
selected_for_scoring = True
else:
selected_for_scoring = False
new_score = DevstarScore(experiment=get_experiment(
legacy_row[0], legacy_row[1]),
area_adult=legacy_row[5],
area_larva=legacy_row[6],
area_embryo=legacy_row[7],
count_adult=count_adult,
count_larva=count_larva,
is_bacteria_present=legacy_row[16],
selected_for_scoring=selected_for_scoring,
gi_score_larva_per_adult=legacy_row[17],
gi_score_survival=legacy_row[18])
# Clean the object to populate the fields derived from other fields
new_score.clean()
###############
# SANITY CHECKS
###############
errors = []
new_allele = new_score.experiment.worm_strain.allele
if (new_allele != legacy_row[2]):
# Deal with case of legacy database using zc310 instead of zu310
if (legacy_row[2] == 'zc310' and new_allele == 'zu310'):
pass
# Deal with some case sensitivity issues in legacy database
# (e.g. see experiment 32405, where the allele is capitalized).
# Can't do lower() in all cases because "N2" should always be
# capitalized.
elif new_allele == legacy_row[2].lower():
pass
else:
errors.append('allele mismatch')
# Deal with case of some experiments having the wrong RNAiPlateID
# in the legacy database's RawDataWithScore table. This field is
# redundant with the RawData table, and RawData is more trustworthy;
# however it is still worthwhile to perform this check in order
# to find the mismatches, and to confirm manually that each one
# makes sense.
new_lp = new_score.experiment.library_stock.plate_id
legacy_lp = legacy_row[4].replace('_', '-').replace('zc310', 'zu310')
if (legacy_lp != new_lp
and new_score.experiment.plate_id not in (461, 8345)
and ('vidal-' not in new_lp
or legacy_lp != new_lp.split('vidal-')[1])):
errors.append('RNAi plate mismatch: {} {}')
if new_score.count_embryo != legacy_row[10]:
errors.append('embryo count mismatch')
if (new_score.embryo_per_adult and legacy_row[8]
and legacy_row[8] != -1
and int(new_score.embryo_per_adult) != legacy_row[11]):
errors.append('embryo per adult mismatch')
if (new_score.larva_per_adult and legacy_row[9] and legacy_row[9] != -1
and int(new_score.larva_per_adult) != legacy_row[12]):
errors.append('larva per adult mismatch')
if (new_score.survival and not compare_floats_for_equality(
new_score.survival, legacy_row[13]) and legacy_row[13] != 0):
errors.append('invalid survival')
if (new_score.lethality and not compare_floats_for_equality(
new_score.lethality, legacy_row[14]) and legacy_row[13] != 0):
errors.append('invalid lethality')
if errors:
raise CommandError(
'DevstarScore for {}:{} had these errors: {}'.format(
legacy_row[0], legacy_row[1], errors))
return update_or_save_object(
command,
new_score,
recorded_scores,
fields_to_compare,
alternate_pk={'experiment': new_score.experiment})
