def test_find_cached_zip(self):
"""Check that a cached zip is found within the given time"""
# Create a resource
req = {'resource_id': '123', 'hello': 'world'}
resource = ResourceFile(req, self._root, self._tempdir, 60*60*24)
w = resource.get_writer()
w.write('hello world')
resource.create_zip(self._zip)
resource.clean_work_files()
# Now look for it again
resource2 = ResourceFile(req, self._root, self._tempdir, 60*60*24)
assert_true(resource2.zip_file_exists())
def test_not_cached_zip(self):
"""Ensure that a different request doesn't match a cached zip"""
# Create a resource
req = {'resource_id': '123', 'hello': 'world'}
resource = ResourceFile(req, self._root, self._tempdir, 60*60*24)
w = resource.get_writer()
w.write('hello world')
resource.create_zip(self._zip)
resource.clean_work_files()
# Now look for it again
req2 = {'resource_id': '123', 'hello': 'world!'}
resource2 = ResourceFile(req2, self._root, self._tempdir, 60*60*24)
assert_false(resource2.zip_file_exists())
def test_cached_zip_timeout(self):
"""Ensure that a cached zip file is not found if it has timed out"""
# Create a resource
req = {'resource_id': '123', 'hello': 'world'}
resource = ResourceFile(req, self._root, self._tempdir, 60*60*24)
w = resource.get_writer()
w.write('hello world')
resource.create_zip(self._zip)
resource.clean_work_files()
# Sleep, and look for it again
time.sleep(2)
resource2 = ResourceFile(req, self._root, self._tempdir, 1)
assert_false(resource2.zip_file_exists())
def speed(self):
""" Return the task estimated time as either 'fast' or 'slow'.
If the file exists in the cache, then this returns 'fast'. It returns
'slow' otherwise.
"""
resource = ResourceFile(
self.request_params,
self.config['STORE_DIRECTORY'],
self.config['TEMP_DIRECTORY'],
self.config['CACHE_TIME']
)
if resource.zip_file_exists():
return 'fast'
else:
return 'slow'
def speed(self):
""" Return the task estimated time as either 'fast' or 'slow'.
If the file exists in the cache, then this returns 'fast'. It returns
'slow' otherwise.
"""
resource = ResourceFile(
self.request_params,
self.config['STORE_DIRECTORY'],
self.config['TEMP_DIRECTORY'],
self.config['CACHE_TIME']
)
if resource.zip_file_exists():
return 'fast'
else:
return 'slow'
def test_cached_zip_parameters(self):
"""Check that the order of parameters and the email parameter
do not affect the ability to find a cached zip file"""
# Create a resource
req = OrderedDict()
req['resource_id'] = '123'
req['email'] = 'carrot'
req['hello'] = 'world'
resource = ResourceFile(req, self._root, self._tempdir, 60*60*24)
w = resource.get_writer()
w.write('hello world')
resource.create_zip(self._zip)
resource.clean_work_files()
# Now look for it again
req2 = OrderedDict()
req2['email'] = 'cake'
req2['hello'] = 'world'
req2['resource_id'] = '123'
resource2 = ResourceFile(req2, self._root, self._tempdir, 60*60*24)
assert_true(resource2.zip_file_exists())
def _run(self):
"""Run the task"""
self.log.info("Task parameters: {}".format(str(self.request_params)))
# Get/create the file
resource = ResourceFile(
self.request_params,
self.config['STORE_DIRECTORY'],
self.config['TEMP_DIRECTORY'],
self.config['CACHE_TIME']
)
if not resource.zip_file_exists():
self.create_zip(resource)
else:
self.log.info("Found file in cache")
zip_file_name = resource.get_zip_file_name()
self.log.info("Got ZIP file {}. Emailing link.".format(zip_file_name))
# Email the link
place_holders = {
'resource_id': self.request_params['resource_id'],
'zip_file_name': os.path.basename(zip_file_name),
'ckan_host': self.host(),
# retrieve a doi from the request params, if there is one, otherwise default to the empty string
'doi': self.request_params.get('doi', ''),
# default the doi_body to the empty string, we'll fill it in below if necessary
'doi_body': '',
}
# if we have the DOI_BODY config option and a doi in the place holders, better load it up
if 'DOI_BODY' in self.config and place_holders['doi']:
place_holders['doi_body'] = self.config['DOI_BODY'].format(**place_holders)
from_addr = self.config['EMAIL_FROM'].format(**place_holders)
msg = MIMEText(self.config['EMAIL_BODY'].format(**place_holders))
msg['Subject'] = self.config['EMAIL_SUBJECT'].format(**place_holders)
msg['From'] = from_addr
msg['To'] = self.request_params['email']
server = smtplib.SMTP(self.config['SMTP_HOST'], self.config['SMTP_PORT'])
try:
if 'SMTP_LOGIN' in self.config:
server.login(self.config['SMTP_LOGIN'], self.config['SMTP_PASSWORD'])
server.sendmail(from_addr, self.request_params['email'], msg.as_string())
finally:
server.quit()
def test_writer(self):
"""Test that writen data is correct"""
# Create a resource
req = {'resource_id': '123', 'hello': 'world'}
resource = ResourceFile(req, self._root, self._tempdir, 60*60*24)
w = resource.get_writer('one.txt')
w.write('hello world')
resource.create_zip(self._zip)
resource.clean_work_files()
# Now check it's content
p = subprocess.Popen(
['unzip', '-p', resource.get_zip_file_name()],
stdout=subprocess.PIPE
)
assert_equals('hello world', "".join(p.stdout.readlines()))
def test_request_id_default_file_name(self):
"""Test that default file names based on request_id work"""
# Create a resource
req = {'resource_id': 'abc123efgwow', 'hello': 'world'}
resource = ResourceFile(req, self._root, self._tempdir, 60*60*24)
w = resource.get_writer()
w.write('hello world')
resource.create_zip(self._zip)
resource.clean_work_files()
# Sleep, and look for it again
p = subprocess.Popen(
['unzip', '-l', resource.get_zip_file_name()],
stdout=subprocess.PIPE
)
out = " ".join(p.stdout.readlines())
assert_in('abc123efgwow', out)
def test_clean_up(self):
"""Ensure that work files are cleaned up"""
# Create a resource
req = {'resource_id': '123', 'hello': 'world'}
resource = ResourceFile(req, self._root, self._tempdir, 60*60*24)
w = resource.get_writer()
w.write('hello world')
resource.create_zip(self._zip)
resource.clean_work_files()
# Ensure writers are closed
assert_true(w.closed)
# Ensure work folder has been removed
assert_equals([], os.listdir(self._tempdir))
def test_csv_writer(self):
"""Test that csv data is correct"""
# Create a resource
req = {'resource_id': '123', 'hello': 'world'}
resource = ResourceFile(req, self._root, self._tempdir, 60*60*24)
w = resource.get_csv_writer('one.csv')
w.writerows([['one', 'two'],['hello', 'world']])
resource.create_zip(self._zip)
resource.clean_work_files()
# Now check it's content
p = subprocess.Popen(
['unzip', '-p', resource.get_zip_file_name()],
stdout=subprocess.PIPE
)
assert_equals(
"one,two\nhello,world",
''.join(p.stdout.readlines()).strip()
)
def test_request_url_default_file_name(self):
"""Test that default file names based on request_id work"""
# Create a resource
req = {
'resource_id': '1',
'resource_url': 'http://somewhere.com/hellow/world.png?one=two'
}
resource = ResourceFile(req, self._root, self._tempdir, 60*60*24)
w = resource.get_writer()
w.write('hello world')
resource.create_zip(self._zip)
resource.clean_work_files()
# Sleep, and look for it again
p = subprocess.Popen(
['unzip', '-l', resource.get_zip_file_name()],
stdout=subprocess.PIPE
)
out = " ".join(p.stdout.readlines())
assert_in('world.png', out)
assert_not_in('world.png?one=two', out)
def test_zip_file_created(self):
"""Create a simple resource with one file, and make sure the zip file
is created as expected.
This uses get_writer, but doesn't test if the written file is in the
archive, let alone what it's content may be
"""
# Create the resource
req = {'resource_id': '123'}
resource = ResourceFile(req, self._root, self._tempdir, 60*60*24)
w = resource.get_writer()
w.write('hello world')
resource.create_zip(self._zip)
resource.clean_work_files()
# Test the zip archive exists as expected
assert_true(resource.zip_file_exists())
assert_true(os.path.exists(resource.get_zip_file_name()))
base, ext = os.path.splitext(resource.get_zip_file_name())
assert_equals('.zip', ext)
res = subprocess.call(['unzip', '-qq', '-t', resource.get_zip_file_name()])
assert_equals(0, res)
def test_zip_file_contains_files(self):
"""Test that the zip files contains expected files
Note that this uses the writer and csv writer, but only tests
if the files exists - not their content.
"""
# Create the resource
req = {'resource_id': '123'}
resource = ResourceFile(req, self._root, self._tempdir, 60*60*24)
w1 = resource.get_writer('one.txt')
w1.write('hello world')
w2 = resource.get_writer('two.txt')
w2.write('hello again')
w3 = resource.get_csv_writer('one.csv')
w3.writerow(['hello', 'world'])
w4 = resource.get_csv_writer('two.csv')
w4.writerow(['hello', 'world'])
resource.create_zip(self._zip)
resource.clean_work_files()
# Test the zip archive contains the expected files
p = subprocess.Popen(
['unzip', '-l', resource.get_zip_file_name()],
stdout=subprocess.PIPE
)
out = " ".join(p.stdout.readlines())
assert_in('one.txt', out)
assert_in('two.txt', out)
assert_in('one.csv', out)
assert_in('two.csv', out)