def upload(self, data, sli_manifest):
'''Create zip file with data in csv format and manifest file, then create
directory in webdav and upload the zip file there.
@param data: csv data to upload
@param sli_manifest: dictionary with the columns definitions
@param wait_for_finish: check periodically for the integration result
return the name of the temporary file, hence the name of the directory
created in webdav uploads folder
'''
#print '\nupload data:'
#print data
#print '\nupload sli_manifest:'
#print sli_manifest
filename = create_archive(data, sli_manifest)
dir_name = os.path.basename(filename)
self.request(self.UPLOADS_URI % dir_name, method='MKCOL')
f = open(filename, 'rb')
# can it be streamed?
self.request(''.join(
(self.UPLOADS_URI % dir_name, DEFAULT_ARCHIVE_NAME)),
data=f.read(),
headers={'Content-Type': 'application/zip'},
method='PUT')
f.close()
#print '\nfilename:',filename
os.remove(filename)
return dir_name
def upload(self, data, sli_manifest):
'''Create zip file with data in csv format and manifest file, then create
directory in webdav and upload the zip file there.
@param data: csv data to upload
@param sli_manifest: dictionary with the columns definitions
@param wait_for_finish: check periodically for the integration result
return the name of the temporary file, hence the name of the directory
created in webdav uploads folder
'''
#print '\nupload data:'
#print data
#print '\nupload sli_manifest:'
#print sli_manifest
filename = create_archive(data, sli_manifest)
dir_name = os.path.basename(filename)
self.request(self.UPLOADS_URI % dir_name, method='MKCOL')
f = open(filename, 'rb')
# can it be streamed?
self.request(''.join((self.UPLOADS_URI % dir_name, DEFAULT_ARCHIVE_NAME)),
data=f.read(), headers={'Content-Type': 'application/zip'},
method='PUT')
f.close()
#print '\nfilename:',filename
os.remove(filename)
return dir_name
def test_archive(self):
for (example, ExampleDataset) in examples.examples:
filename = create_archive(example.data_csv, example.sli_manifest)
zip_file = ZipFile(filename, "r")
self.assertEquals(None, zip_file.testzip())
self.assertEquals(zip_file.namelist(), ['data.csv', 'upload_info.json'])
zip_file.close()
os.remove(filename)
def test_keep_csv(self):
csv_file = os.path.join(os.path.abspath('./'), 'tmp.csv')
for (example, ExampleDataset) in examples.examples:
filename = create_archive(
example.data_csv, example.sli_manifest, [], [],
keep_csv=True, csv_file=csv_file
)
with open(csv_file, 'r') as f:
content = f.read()
os.remove(csv_file)
self.assertListEqual(csv_to_list(example.data_csv), csv_to_list(content))
self.assertRaises(
TypeError, create_archive, example.data_csv,
example.sli_manifest, [], [], keep_csv=True
)
def upload(
self, data, sli_manifest, dates=[], datetimes=[],
keep_csv=False, csv_file=None, no_upload=False,
csv_input_path=None
):
'''Create zip file with data in csv format and manifest file, then create
directory in webdav and upload the zip file there.
@param data: csv data to upload
@param sli_manifest: dictionary with the columns definitions
@param dates: list of date fields
@param datetimes: lits of datetime fields
@param keep_csv: keep csv file on filesystem
@param csv_file: abspath where to keep csv file
@param no_upload: do the upload or not
@param csv_input_path: push data from a csv instead of
data from the `data` parameter
return the name of the temporary file, hence the name of the directory
created in webdav uploads folder
'''
archive = create_archive(
data, sli_manifest, dates, datetimes,
keep_csv, csv_file, csv_input_path
)
if no_upload:
os.remove(archive)
return csv_file
dir_name = os.path.basename(archive)
# create the folder on WebDav
self.mkcol(uri=self.UPLOADS_URI % dir_name)
# open the files to read them
f_archive = open(archive, 'rb')
# upload the files to WebDav
archive_uri = ''.join((self.UPLOADS_URI % dir_name, DEFAULT_ARCHIVE_NAME))
self.put(uri=archive_uri, data=f_archive.read(),
headers={'Content-Type': 'application/zip'})
# close and remove the files
f_archive.close()
os.remove(archive)
return dir_name