def _update_data():
"""
Requests a data transfer from Globus/Ivy Service
Updates the database based on local files placed by IVY."""
app.logger.info("Executing Update Data Task")
ivy_service = IvyService()
ivy_service.request_transfer()
load_local_files()
def update_data():
"""Updates the database based on local files placed by IVY and recoreds
read in from the firecloud database."""
fb_service = FirebaseService()
ivy_service = IvyService()
samples = fb_service.get_samples()
samples.extend(ivy_service.load_directory())
SampleService().add_or_update_records(samples)
def load_local_files():
"""
Just process any files that are local to the system.
"""
ivy_service = IvyService()
files, samples = ivy_service.load_directory()
SampleService().add_or_update_records(samples)
for file in files:
db.session.add(file)
db.session.commit()
if app.config['DELETE_IVY_FILES']:
ivy_service.delete_file(file.file_name)
else:
app.logger.info("Not Deleting Files, per DELETE_IVY_FILES flag")
db.session.commit()
def test_load_directory(self):
self.assertEqual(0, db.session.query(IvyFile).count())
app.config['IVY_IMPORT_DIR'] = os.path.join(app.root_path, '..',
'tests', 'data',
'import_directory')
files, _ = IvyService().load_directory()
self.assertEqual(4, len(files))
def test_invalid_date(self):
"""If a record with an unparssable date comes through, use today's date in the date field."""
ivy_incorrect_path = os.path.join(app.root_path, '..', 'tests', 'data')
records = IvyService.samples_from_ivy_file(ivy_incorrect_path,
'incorrect_date.csv')
self.assertEquals(4, len(records))
self.assertEquals('987655321-TN-20212719-4321', records[2].barcode)
def test_correlate_samples_ivy_first(self):
service = SampleService()
self.assertEqual(0, len(db.session.query(Sample).all()))
ivy_samples = IvyService.samples_from_ivy_file(self.ivy_file)
service.add_or_update_records(ivy_samples)
self.assertEqual(6, len(db.session.query(Sample).all()))
fb_samples = self.get_firebase_records()
service.add_or_update_records(fb_samples)
self.assertEqual(
6,
len(db.session.query(Sample).filter(Sample.in_ivy == True).all()))
self.assertEqual(
4,
len(
db.session.query(Sample).filter(
Sample.in_firebase == True).all()))
self.assertEqual(
3,
len(
db.session.query(Sample).filter(
Sample.in_firebase == True).filter(
Sample.in_ivy == True).all()))
self.assertEqual(7, len(db.session.query(Sample).all()))
def test_correlate_samples_firebase_first(self):
"""Load up all the samples from firebase, then load data from ivy, and assure we
get the correct number of records with the right details."""
# Load firebase records
service = SampleService()
fb_samples = self.get_firebase_records()
self.assertEqual(0, len(db.session.query(Sample).all()))
service.add_or_update_records(fb_samples)
self.assertEqual(4, len(db.session.query(Sample).all()))
ivy_samples = IvyService.samples_from_ivy_file(self.ivy_file)
service.add_or_update_records(ivy_samples)
# There are 6 records in ivy, but three records that should match up, giving seven total
self.assertEqual(
6,
len(db.session.query(Sample).filter(Sample.in_ivy == True).all()))
self.assertEqual(
4,
len(
db.session.query(Sample).filter(
Sample.in_firebase == True).all()))
self.assertEqual(
3,
len(
db.session.query(Sample).filter(
Sample.in_firebase == True).filter(
Sample.in_ivy == True).all()))
self.assertEqual(7, len(db.session.query(Sample).all()))
def test_timezone_offset(self):
"""The date and time returned from the lab / Globus is in EST, be sure to save it as such to
avoid a 5 hour offset, when it is assumed to be in GMT."""
records = IvyService.samples_from_ivy_file(self.ivy_path,
self.ivy_file)
self.assertEqual("987654321", records[0].student_id)
self.assertIsNotNone(records[0].date.tzinfo,
"on ingestion, the date should be in EST")
# original date "202009030809"
date_string = '202009031209' # UTC is 4 hours head for this date
date = datetime.datetime.strptime(date_string, '%Y%m%d%H%M')
db.session.add(records[0])
db.session.commit()
self.assertEqual(date, records[0].date)
def delete():
from communicator.services.ivy_service import IvyService
ivy_service = IvyService()
ivy_service.delete_file()
def transfer():
from communicator.services.ivy_service import IvyService
ivy_service = IvyService()
ivy_service.request_transfer()
def list_files():
from communicator.services.ivy_service import IvyService
ivy_service = IvyService()
ivy_service.list_files()
def globus_token():
from communicator.services.ivy_service import IvyService
ivy_service = IvyService()
ivy_service.get_access_token()
def test_invalid_file(self):
with self.assertRaises(CommError):
ivy_incorrect_path = os.path.join(app.root_path, '..', 'tests',
'data')
IvyService.samples_from_ivy_file(ivy_incorrect_path,
'incorrect.csv')
def test_read_file_and_build_records(self):
records = IvyService.samples_from_ivy_file(self.ivy_path,
self.ivy_file)
self.assertEqual("987654321", records[0].student_id)
self.assertEqual("*****@*****.**", records[1].email)
self.assertEqual("1142270225", records[2].result_code)
def count_files_in_ivy():
from communicator.services.ivy_service import IvyService
ivy_service = IvyService()
count = ivy_service.get_file_count_from_globus()
print(f"There are {count} files awaiting transfer")