def handle(self, *args, **options): """ Sets options common to all commands. Any command subclassing this object should implement its own handle method, as is standard in Django, and run this method via a super call to inherit its functionality. """ # Set global options self.verbosity = options.get("verbosity") self.no_color = options.get("no_color") # set up data directories self.data_dir = get_data_directory() self.tsv_dir = os.path.join(self.data_dir, 'tsv') self.csv_dir = os.path.join(self.data_dir, 'csv') os.path.exists(self.data_dir) or os.makedirs(self.data_dir) os.path.exists(self.tsv_dir) or os.makedirs(self.tsv_dir) os.path.exists(self.csv_dir) or os.makedirs(self.csv_dir) # set path where zip will be downloaded self.download_dir = os.path.join(self.data_dir, 'download') self.zip_path = os.path.join(self.download_dir, self.url.split('/')[-1]) # Start the clock self.start_datetime = datetime.now()
def set_global_options(self, options): """ Set options to all commands. """ # Set global options self.verbosity = options.get("verbosity") self.no_color = options.get("no_color") # set up data directories self.data_dir = get_data_directory() self.tsv_dir = os.path.join(self.data_dir, 'tsv') self.csv_dir = os.path.join(self.data_dir, 'csv') os.path.exists(self.data_dir) or os.makedirs(self.data_dir) os.path.exists(self.tsv_dir) or os.makedirs(self.tsv_dir) os.path.exists(self.csv_dir) or os.makedirs(self.csv_dir) # set path where zip will be downloaded self.download_dir = os.path.join(self.data_dir, 'download') self.zip_path = os.path.join( self.download_dir, self.url.split('/')[-1] ) # Start the clock self.start_datetime = datetime.now()
def get_tsv_path(self): """ Returns the path to the model's raw TSV data file. """ return os.path.join( calaccess_raw.get_data_directory(), 'tsv', self.get_tsv_name() )
def handle(self, *args, **options): """ Make it happen. """ super(Command, self).handle(*args, **options) # Parse model name self.model_name = options['model_name'] # Log out what we're doing ... self.log(" Archiving %s.csv" % self.model_name) # ... get the current version ... version = ProcessedDataVersion.objects.latest('process_start_datetime') # ... and the processed file object ... try: processed_file = version.files.get(file_name=self.model_name) except ProcessedDataFile.DoesNotExist: processed_file = version.files.create(file_name=self.model_name) # Get the data obj that is paired with the processed_file obj data_model = self.get_model(processed_file) # Update the records count processed_file.records_count = data_model.objects.count() # Save it processed_file.save() # Remove previous .CSV files processed_file.file_archive.delete() # Figure out the path where we will save the file csv_dir = os.path.join(get_data_directory(), 'processed', data_model().klass_group.lower()) os.path.exists(csv_dir) or os.mkdir(csv_dir) csv_name = '{}.csv'.format(processed_file.file_name) csv_path = os.path.join(csv_dir, csv_name) # Export a new one try: copy_to_fields = tuple(i[0] for i in data_model.copy_to_fields) except AttributeError: copy_to_fields = tuple() data_model.objects.to_csv(csv_path, *copy_to_fields) # Open up the .CSV file for reading so we can wrap it in the Django File obj with open(csv_path, 'rb') as csv_file: # Save the .CSV on the processed data file processed_file.file_archive.save(csv_name, File(csv_file)) # Save it to the model processed_file.file_size = os.path.getsize(csv_path) processed_file.save()
def handle(self, *args, **options): """ Sets options common to all commands. Any command subclassing this object should implement its own handle method, as is standard in Django, and run this method via a super call to inherit its functionality. """ # Set global options self.verbosity = options.get("verbosity") self.no_color = options.get("no_color") # Start the clock self.start_datetime = timezone.now() # set up processed data directory self.data_dir = get_data_directory() self.processed_data_dir = os.path.join( self.data_dir, 'processed', ) if not os.path.exists(self.processed_data_dir): # make the processed data director os.makedirs(self.processed_data_dir)
def test_dir_basedir(self): """ Tests for directory functions __init__.py file with different settings. """ calaccess_raw.get_data_directory()
def test_dir_configured(self): """ Tests for directory functions __init__.py file. """ calaccess_raw.get_data_directory()
def test_dir_errors(self): """ Test error expected when download directory is missing. """ with self.assertRaises(ValueError): calaccess_raw.get_data_directory()
def get_tsv_path(self): """ Returns the path to the model's raw TSV data file. """ return os.path.join(calaccess_raw.get_data_directory(), 'tsv', self.get_tsv_name())