def upload(self, data, sli_manifest): '''Create zip file with data in csv format and manifest file, then create directory in webdav and upload the zip file there. @param data: csv data to upload @param sli_manifest: dictionary with the columns definitions @param wait_for_finish: check periodically for the integration result return the name of the temporary file, hence the name of the directory created in webdav uploads folder ''' #print '\nupload data:' #print data #print '\nupload sli_manifest:' #print sli_manifest filename = create_archive(data, sli_manifest) dir_name = os.path.basename(filename) self.request(self.UPLOADS_URI % dir_name, method='MKCOL') f = open(filename, 'rb') # can it be streamed? self.request(''.join( (self.UPLOADS_URI % dir_name, DEFAULT_ARCHIVE_NAME)), data=f.read(), headers={'Content-Type': 'application/zip'}, method='PUT') f.close() #print '\nfilename:',filename os.remove(filename) return dir_name
def upload(self, data, sli_manifest): '''Create zip file with data in csv format and manifest file, then create directory in webdav and upload the zip file there. @param data: csv data to upload @param sli_manifest: dictionary with the columns definitions @param wait_for_finish: check periodically for the integration result return the name of the temporary file, hence the name of the directory created in webdav uploads folder ''' #print '\nupload data:' #print data #print '\nupload sli_manifest:' #print sli_manifest filename = create_archive(data, sli_manifest) dir_name = os.path.basename(filename) self.request(self.UPLOADS_URI % dir_name, method='MKCOL') f = open(filename, 'rb') # can it be streamed? self.request(''.join((self.UPLOADS_URI % dir_name, DEFAULT_ARCHIVE_NAME)), data=f.read(), headers={'Content-Type': 'application/zip'}, method='PUT') f.close() #print '\nfilename:',filename os.remove(filename) return dir_name
def test_archive(self): for (example, ExampleDataset) in examples.examples: filename = create_archive(example.data_csv, example.sli_manifest) zip_file = ZipFile(filename, "r") self.assertEquals(None, zip_file.testzip()) self.assertEquals(zip_file.namelist(), ['data.csv', 'upload_info.json']) zip_file.close() os.remove(filename)
def test_keep_csv(self): csv_file = os.path.join(os.path.abspath('./'), 'tmp.csv') for (example, ExampleDataset) in examples.examples: filename = create_archive( example.data_csv, example.sli_manifest, [], [], keep_csv=True, csv_file=csv_file ) with open(csv_file, 'r') as f: content = f.read() os.remove(csv_file) self.assertListEqual(csv_to_list(example.data_csv), csv_to_list(content)) self.assertRaises( TypeError, create_archive, example.data_csv, example.sli_manifest, [], [], keep_csv=True )
def upload( self, data, sli_manifest, dates=[], datetimes=[], keep_csv=False, csv_file=None, no_upload=False, csv_input_path=None ): '''Create zip file with data in csv format and manifest file, then create directory in webdav and upload the zip file there. @param data: csv data to upload @param sli_manifest: dictionary with the columns definitions @param dates: list of date fields @param datetimes: lits of datetime fields @param keep_csv: keep csv file on filesystem @param csv_file: abspath where to keep csv file @param no_upload: do the upload or not @param csv_input_path: push data from a csv instead of data from the `data` parameter return the name of the temporary file, hence the name of the directory created in webdav uploads folder ''' archive = create_archive( data, sli_manifest, dates, datetimes, keep_csv, csv_file, csv_input_path ) if no_upload: os.remove(archive) return csv_file dir_name = os.path.basename(archive) # create the folder on WebDav self.mkcol(uri=self.UPLOADS_URI % dir_name) # open the files to read them f_archive = open(archive, 'rb') # upload the files to WebDav archive_uri = ''.join((self.UPLOADS_URI % dir_name, DEFAULT_ARCHIVE_NAME)) self.put(uri=archive_uri, data=f_archive.read(), headers={'Content-Type': 'application/zip'}) # close and remove the files f_archive.close() os.remove(archive) return dir_name