def handle(self, *args, **options):
"""
Sets options common to all commands.
Any command subclassing this object should implement its own
handle method, as is standard in Django, and run this method
via a super call to inherit its functionality.
"""
# Set global options
self.verbosity = options.get("verbosity")
self.no_color = options.get("no_color")
# set up data directories
self.data_dir = get_data_directory()
self.tsv_dir = os.path.join(self.data_dir, 'tsv')
self.csv_dir = os.path.join(self.data_dir, 'csv')
os.path.exists(self.data_dir) or os.makedirs(self.data_dir)
os.path.exists(self.tsv_dir) or os.makedirs(self.tsv_dir)
os.path.exists(self.csv_dir) or os.makedirs(self.csv_dir)
# set path where zip will be downloaded
self.download_dir = os.path.join(self.data_dir, 'download')
self.zip_path = os.path.join(self.download_dir,
self.url.split('/')[-1])
# Start the clock
self.start_datetime = datetime.now()
def set_global_options(self, options):
"""
Set options to all commands.
"""
# Set global options
self.verbosity = options.get("verbosity")
self.no_color = options.get("no_color")
# set up data directories
self.data_dir = get_data_directory()
self.tsv_dir = os.path.join(self.data_dir, 'tsv')
self.csv_dir = os.path.join(self.data_dir, 'csv')
os.path.exists(self.data_dir) or os.makedirs(self.data_dir)
os.path.exists(self.tsv_dir) or os.makedirs(self.tsv_dir)
os.path.exists(self.csv_dir) or os.makedirs(self.csv_dir)
# set path where zip will be downloaded
self.download_dir = os.path.join(self.data_dir, 'download')
self.zip_path = os.path.join(
self.download_dir,
self.url.split('/')[-1]
)
# Start the clock
self.start_datetime = datetime.now()
def get_tsv_path(self):
"""
Returns the path to the model's raw TSV data file.
"""
return os.path.join(
calaccess_raw.get_data_directory(),
'tsv',
self.get_tsv_name()
)
def handle(self, *args, **options):
"""
Make it happen.
"""
super(Command, self).handle(*args, **options)
# Parse model name
self.model_name = options['model_name']
# Log out what we're doing ...
self.log(" Archiving %s.csv" % self.model_name)
# ... get the current version ...
version = ProcessedDataVersion.objects.latest('process_start_datetime')
# ... and the processed file object ...
try:
processed_file = version.files.get(file_name=self.model_name)
except ProcessedDataFile.DoesNotExist:
processed_file = version.files.create(file_name=self.model_name)
# Get the data obj that is paired with the processed_file obj
data_model = self.get_model(processed_file)
# Update the records count
processed_file.records_count = data_model.objects.count()
# Save it
processed_file.save()
# Remove previous .CSV files
processed_file.file_archive.delete()
# Figure out the path where we will save the file
csv_dir = os.path.join(get_data_directory(), 'processed',
data_model().klass_group.lower())
os.path.exists(csv_dir) or os.mkdir(csv_dir)
csv_name = '{}.csv'.format(processed_file.file_name)
csv_path = os.path.join(csv_dir, csv_name)
# Export a new one
try:
copy_to_fields = tuple(i[0] for i in data_model.copy_to_fields)
except AttributeError:
copy_to_fields = tuple()
data_model.objects.to_csv(csv_path, *copy_to_fields)
# Open up the .CSV file for reading so we can wrap it in the Django File obj
with open(csv_path, 'rb') as csv_file:
# Save the .CSV on the processed data file
processed_file.file_archive.save(csv_name, File(csv_file))
# Save it to the model
processed_file.file_size = os.path.getsize(csv_path)
processed_file.save()
def handle(self, *args, **options):
"""
Sets options common to all commands.
Any command subclassing this object should implement its own
handle method, as is standard in Django, and run this method
via a super call to inherit its functionality.
"""
# Set global options
self.verbosity = options.get("verbosity")
self.no_color = options.get("no_color")
# Start the clock
self.start_datetime = timezone.now()
# set up processed data directory
self.data_dir = get_data_directory()
self.processed_data_dir = os.path.join(
self.data_dir,
'processed',
)
if not os.path.exists(self.processed_data_dir):
# make the processed data director
os.makedirs(self.processed_data_dir)
def test_dir_basedir(self):
"""
Tests for directory functions __init__.py file with different settings.
"""
calaccess_raw.get_data_directory()
def test_dir_configured(self):
"""
Tests for directory functions __init__.py file.
"""
calaccess_raw.get_data_directory()
def test_dir_errors(self):
"""
Test error expected when download directory is missing.
"""
with self.assertRaises(ValueError):
calaccess_raw.get_data_directory()
def get_tsv_path(self):
"""
Returns the path to the model's raw TSV data file.
"""
return os.path.join(calaccess_raw.get_data_directory(), 'tsv',
self.get_tsv_name())
def test_dir_basedir(self):
"""
Tests for directory functions __init__.py file with different settings.
"""
calaccess_raw.get_data_directory()
def test_dir_configured(self):
"""
Tests for directory functions __init__.py file.
"""
calaccess_raw.get_data_directory()
def test_dir_errors(self):
"""
Test error expected when download directory is missing.
"""
with self.assertRaises(ValueError):
calaccess_raw.get_data_directory()