def test_find_cached_zip(self): """Check that a cached zip is found within the given time""" # Create a resource req = {'resource_id': '123', 'hello': 'world'} resource = ResourceFile(req, self._root, self._tempdir, 60*60*24) w = resource.get_writer() w.write('hello world') resource.create_zip(self._zip) resource.clean_work_files() # Now look for it again resource2 = ResourceFile(req, self._root, self._tempdir, 60*60*24) assert_true(resource2.zip_file_exists())
def test_not_cached_zip(self): """Ensure that a different request doesn't match a cached zip""" # Create a resource req = {'resource_id': '123', 'hello': 'world'} resource = ResourceFile(req, self._root, self._tempdir, 60*60*24) w = resource.get_writer() w.write('hello world') resource.create_zip(self._zip) resource.clean_work_files() # Now look for it again req2 = {'resource_id': '123', 'hello': 'world!'} resource2 = ResourceFile(req2, self._root, self._tempdir, 60*60*24) assert_false(resource2.zip_file_exists())
def test_cached_zip_timeout(self): """Ensure that a cached zip file is not found if it has timed out""" # Create a resource req = {'resource_id': '123', 'hello': 'world'} resource = ResourceFile(req, self._root, self._tempdir, 60*60*24) w = resource.get_writer() w.write('hello world') resource.create_zip(self._zip) resource.clean_work_files() # Sleep, and look for it again time.sleep(2) resource2 = ResourceFile(req, self._root, self._tempdir, 1) assert_false(resource2.zip_file_exists())
def speed(self): """ Return the task estimated time as either 'fast' or 'slow'. If the file exists in the cache, then this returns 'fast'. It returns 'slow' otherwise. """ resource = ResourceFile( self.request_params, self.config['STORE_DIRECTORY'], self.config['TEMP_DIRECTORY'], self.config['CACHE_TIME'] ) if resource.zip_file_exists(): return 'fast' else: return 'slow'
def test_cached_zip_parameters(self): """Check that the order of parameters and the email parameter do not affect the ability to find a cached zip file""" # Create a resource req = OrderedDict() req['resource_id'] = '123' req['email'] = 'carrot' req['hello'] = 'world' resource = ResourceFile(req, self._root, self._tempdir, 60*60*24) w = resource.get_writer() w.write('hello world') resource.create_zip(self._zip) resource.clean_work_files() # Now look for it again req2 = OrderedDict() req2['email'] = 'cake' req2['hello'] = 'world' req2['resource_id'] = '123' resource2 = ResourceFile(req2, self._root, self._tempdir, 60*60*24) assert_true(resource2.zip_file_exists())
def _run(self): """Run the task""" self.log.info("Task parameters: {}".format(str(self.request_params))) # Get/create the file resource = ResourceFile( self.request_params, self.config['STORE_DIRECTORY'], self.config['TEMP_DIRECTORY'], self.config['CACHE_TIME'] ) if not resource.zip_file_exists(): self.create_zip(resource) else: self.log.info("Found file in cache") zip_file_name = resource.get_zip_file_name() self.log.info("Got ZIP file {}. Emailing link.".format(zip_file_name)) # Email the link place_holders = { 'resource_id': self.request_params['resource_id'], 'zip_file_name': os.path.basename(zip_file_name), 'ckan_host': self.host(), # retrieve a doi from the request params, if there is one, otherwise default to the empty string 'doi': self.request_params.get('doi', ''), # default the doi_body to the empty string, we'll fill it in below if necessary 'doi_body': '', } # if we have the DOI_BODY config option and a doi in the place holders, better load it up if 'DOI_BODY' in self.config and place_holders['doi']: place_holders['doi_body'] = self.config['DOI_BODY'].format(**place_holders) from_addr = self.config['EMAIL_FROM'].format(**place_holders) msg = MIMEText(self.config['EMAIL_BODY'].format(**place_holders)) msg['Subject'] = self.config['EMAIL_SUBJECT'].format(**place_holders) msg['From'] = from_addr msg['To'] = self.request_params['email'] server = smtplib.SMTP(self.config['SMTP_HOST'], self.config['SMTP_PORT']) try: if 'SMTP_LOGIN' in self.config: server.login(self.config['SMTP_LOGIN'], self.config['SMTP_PASSWORD']) server.sendmail(from_addr, self.request_params['email'], msg.as_string()) finally: server.quit()
def test_writer(self): """Test that writen data is correct""" # Create a resource req = {'resource_id': '123', 'hello': 'world'} resource = ResourceFile(req, self._root, self._tempdir, 60*60*24) w = resource.get_writer('one.txt') w.write('hello world') resource.create_zip(self._zip) resource.clean_work_files() # Now check it's content p = subprocess.Popen( ['unzip', '-p', resource.get_zip_file_name()], stdout=subprocess.PIPE ) assert_equals('hello world', "".join(p.stdout.readlines()))
def test_request_id_default_file_name(self): """Test that default file names based on request_id work""" # Create a resource req = {'resource_id': 'abc123efgwow', 'hello': 'world'} resource = ResourceFile(req, self._root, self._tempdir, 60*60*24) w = resource.get_writer() w.write('hello world') resource.create_zip(self._zip) resource.clean_work_files() # Sleep, and look for it again p = subprocess.Popen( ['unzip', '-l', resource.get_zip_file_name()], stdout=subprocess.PIPE ) out = " ".join(p.stdout.readlines()) assert_in('abc123efgwow', out)
def test_clean_up(self): """Ensure that work files are cleaned up""" # Create a resource req = {'resource_id': '123', 'hello': 'world'} resource = ResourceFile(req, self._root, self._tempdir, 60*60*24) w = resource.get_writer() w.write('hello world') resource.create_zip(self._zip) resource.clean_work_files() # Ensure writers are closed assert_true(w.closed) # Ensure work folder has been removed assert_equals([], os.listdir(self._tempdir))
def test_csv_writer(self): """Test that csv data is correct""" # Create a resource req = {'resource_id': '123', 'hello': 'world'} resource = ResourceFile(req, self._root, self._tempdir, 60*60*24) w = resource.get_csv_writer('one.csv') w.writerows([['one', 'two'],['hello', 'world']]) resource.create_zip(self._zip) resource.clean_work_files() # Now check it's content p = subprocess.Popen( ['unzip', '-p', resource.get_zip_file_name()], stdout=subprocess.PIPE ) assert_equals( "one,two\nhello,world", ''.join(p.stdout.readlines()).strip() )
def test_request_url_default_file_name(self): """Test that default file names based on request_id work""" # Create a resource req = { 'resource_id': '1', 'resource_url': 'http://somewhere.com/hellow/world.png?one=two' } resource = ResourceFile(req, self._root, self._tempdir, 60*60*24) w = resource.get_writer() w.write('hello world') resource.create_zip(self._zip) resource.clean_work_files() # Sleep, and look for it again p = subprocess.Popen( ['unzip', '-l', resource.get_zip_file_name()], stdout=subprocess.PIPE ) out = " ".join(p.stdout.readlines()) assert_in('world.png', out) assert_not_in('world.png?one=two', out)
def test_zip_file_created(self): """Create a simple resource with one file, and make sure the zip file is created as expected. This uses get_writer, but doesn't test if the written file is in the archive, let alone what it's content may be """ # Create the resource req = {'resource_id': '123'} resource = ResourceFile(req, self._root, self._tempdir, 60*60*24) w = resource.get_writer() w.write('hello world') resource.create_zip(self._zip) resource.clean_work_files() # Test the zip archive exists as expected assert_true(resource.zip_file_exists()) assert_true(os.path.exists(resource.get_zip_file_name())) base, ext = os.path.splitext(resource.get_zip_file_name()) assert_equals('.zip', ext) res = subprocess.call(['unzip', '-qq', '-t', resource.get_zip_file_name()]) assert_equals(0, res)
def test_zip_file_contains_files(self): """Test that the zip files contains expected files Note that this uses the writer and csv writer, but only tests if the files exists - not their content. """ # Create the resource req = {'resource_id': '123'} resource = ResourceFile(req, self._root, self._tempdir, 60*60*24) w1 = resource.get_writer('one.txt') w1.write('hello world') w2 = resource.get_writer('two.txt') w2.write('hello again') w3 = resource.get_csv_writer('one.csv') w3.writerow(['hello', 'world']) w4 = resource.get_csv_writer('two.csv') w4.writerow(['hello', 'world']) resource.create_zip(self._zip) resource.clean_work_files() # Test the zip archive contains the expected files p = subprocess.Popen( ['unzip', '-l', resource.get_zip_file_name()], stdout=subprocess.PIPE ) out = " ".join(p.stdout.readlines()) assert_in('one.txt', out) assert_in('two.txt', out) assert_in('one.csv', out) assert_in('two.csv', out)