def test_sftp_origin(sdc_builder, sdc_executor, sftp): """Smoke test SFTP origin. We first create a file on SFTP server and have the SFTP origin stage read it. We then assert its snapshot. The pipeline look like: sftp_ftp_client >> trash """ sftp_file_name = get_random_string(string.ascii_letters, 10) raw_text_data = 'Hello World!' sftp.put_string(os.path.join(sftp.path, sftp_file_name), raw_text_data) builder = sdc_builder.get_pipeline_builder() sftp_ftp_client = builder.add_stage(name='com_streamsets_pipeline_stage_origin_remote_RemoteDownloadDSource') sftp_ftp_client.file_name_pattern = sftp_file_name sftp_ftp_client.data_format = 'TEXT' trash = builder.add_stage('Trash') sftp_ftp_client >> trash sftp_ftp_client_pipeline = builder.build('SFTP Origin Pipeline').configure_for_environment(sftp) sdc_executor.add_pipeline(sftp_ftp_client_pipeline) snapshot = sdc_executor.capture_snapshot(sftp_ftp_client_pipeline, start_pipeline=True).snapshot sdc_executor.stop_pipeline(sftp_ftp_client_pipeline) assert len(snapshot[sftp_ftp_client].output) == 1 assert snapshot[sftp_ftp_client].output[0].field['text'] == raw_text_data # Delete the test SFTP origin file we created transport, client = sftp.client try: client.remove(os.path.join(sftp.path, sftp_file_name)) finally: client.close() transport.close()
def test_sftp_origin(sdc_builder, sdc_executor, sftp): """Smoke test SFTP origin. We first create a file on SFTP server and have the SFTP origin stage read it. We then assert the ingested data using wiretap. The pipeline looks like: sftp_ftp_client >> wiretap """ sftp_file_name = get_random_string(string.ascii_letters, 10) raw_text_data = 'Hello World!' sftp.put_string(os.path.join(sftp.path, sftp_file_name), raw_text_data) builder = sdc_builder.get_pipeline_builder() sftp_ftp_client = builder.add_stage(name='com_streamsets_pipeline_stage_origin_remote_RemoteDownloadDSource') sftp_ftp_client.file_name_pattern = sftp_file_name sftp_ftp_client.data_format = 'TEXT' wiretap = builder.add_wiretap() sftp_ftp_client >> wiretap.destination sftp_ftp_client_pipeline = builder.build('SFTP Origin Pipeline').configure_for_environment(sftp) sdc_executor.add_pipeline(sftp_ftp_client_pipeline) sdc_executor.start_pipeline(sftp_ftp_client_pipeline).wait_for_pipeline_output_records_count(1) sdc_executor.stop_pipeline(sftp_ftp_client_pipeline) assert len(wiretap.output_records) == 1 assert wiretap.output_records[0].field['text'] == raw_text_data # Delete the test SFTP origin file we created transport, client = sftp.client try: client.remove(os.path.join(sftp.path, sftp_file_name)) finally: client.close() transport.close()
def test_sftp_origin_whole_file_to_s3(sdc_builder, sdc_executor, sftp, aws): """This is a test for SDC-11273. First, it creates a large (~6MB) file and puts it on the SFTP server. Then, it creates a pipeline with SFTP origin and S3 destination, with whole file format, and runs until the single record (file) is complete. Then, it asserts the S3 bucket contents are correct. It passes only if the new option ("Disable Read Ahead Stream") is enabled. """ sftp_file_name = get_random_string(string.ascii_letters, 10) + '.txt' raw_text_data = get_random_string(string.printable, 6000000) sftp.put_string(os.path.join(sftp.path, sftp_file_name), raw_text_data) s3_bucket = aws.s3_bucket_name s3_key = f'{S3_BUCKET_PREFIX}/{sftp_file_name}' # Build the pipeline builder = sdc_builder.get_pipeline_builder() sftp_ftp_client = builder.add_stage(name='com_streamsets_pipeline_stage_origin_remote_RemoteDownloadDSource') sftp_ftp_client.file_name_pattern = sftp_file_name sftp_ftp_client.data_format = 'WHOLE_FILE' if Version(sdc_builder.version) >= Version('3.8.2'): # Disable read-ahead stream as workaround for sshj library issues (see SDC-11273). sftp_ftp_client.disable_read_ahead_stream = True s3_destination = builder.add_stage('Amazon S3', type='destination') s3_destination.file_name_expression = "${record:value('/fileInfo/filename')}" s3_destination.set_attributes(bucket=s3_bucket, data_format='WHOLE_FILE', partition_prefix=s3_key) sftp_ftp_client >> s3_destination sftp_to_s3_pipeline = builder.build().configure_for_environment(aws, sftp) sdc_executor.add_pipeline(sftp_to_s3_pipeline) client = aws.s3 try: # start pipeline and run for one record (the file) sdc_executor.start_pipeline(sftp_to_s3_pipeline).wait_for_pipeline_output_records_count(1) sdc_executor.stop_pipeline(sftp_to_s3_pipeline) # assert record count to S3 the size of the objects put list_s3_objs = client.list_objects_v2(Bucket=s3_bucket, Prefix=s3_key) assert len(list_s3_objs['Contents']) == 1 # read data from S3 to assert contents s3_contents = [client.get_object(Bucket=s3_bucket, Key=s3_content['Key'])['Body'].read().decode().strip() for s3_content in list_s3_objs['Contents']] # compare the S3 bucket contents against the original whole file contents assert s3_contents[0] == raw_text_data finally: delete_keys = {'Objects': [{'Key': k['Key']} for k in client.list_objects_v2(Bucket=s3_bucket, Prefix=s3_key)['Contents']]} client.delete_objects(Bucket=s3_bucket, Delete=delete_keys)
def test_sftp_origin_open_files(sdc_builder, sdc_executor, sftp, ssh): """Test SFTP origin to see if it leaves any open files on the SSH server after its pipeline has processed records. We first create a file on SFTP server and have the SFTP origin stage read it. We then check if open files are left after the pipeline processes all data. The pipeline look like: sftp_ftp_client >> trash """ sftp_file_name = get_random_string(string.ascii_letters, 10) raw_text_data = 'Hello World!' logger.debug('Creating file at %s/%s on SFTP server ...', sftp.path, sftp_file_name) sftp.put_string(os.path.join(sftp.path, sftp_file_name), raw_text_data) builder = sdc_builder.get_pipeline_builder() sftp_ftp_client = builder.add_stage( name='com_streamsets_pipeline_stage_origin_remote_RemoteDownloadDSource' ) sftp_ftp_client.file_name_pattern = sftp_file_name sftp_ftp_client.data_format = 'TEXT' trash = builder.add_stage('Trash') sftp_ftp_client >> trash sftp_ftp_client_pipeline = builder.build( 'SFTP Origin open file check Pipeline').configure_for_environment(sftp) sdc_executor.add_pipeline(sftp_ftp_client_pipeline) start_command = sdc_executor.start_pipeline(sftp_ftp_client_pipeline) ssh_client = ssh.client try: # make sure pipeline has processed all data start_command.wait_for_pipeline_output_records_count(1) # since now pipeline has processed all records, make sure it has not left a open stream to the remote file ssh_stdin, ssh_stdout, ssh_stderr = ssh_client.exec_command( f'lsof | grep {sftp_file_name}') lsof_status = ssh_stdout.channel.recv_exit_status() assert lsof_status == 1 finally: sdc_executor.stop_pipeline(sftp_ftp_client_pipeline) ssh_client.close() # Delete the test SFTP origin file we created transport, sftp_client = sftp.client try: logger.debug('Removing file at %s/%s on SFTP server ...', sftp.path, sftp_file_name) sftp_client.remove(os.path.join(sftp.path, sftp_file_name)) finally: sftp_client.close() transport.close()
def test_sftp_origin_whole_file_to_s3_no_read_permission(sdc_builder, sdc_executor, sftp): """This is a test for SDC-14867. It creates a file with no read permissions and creates one more file with read permissions, when the pipeline runs we will start ingesting from the second file and first file is skipped and an error is reported. We also drop another file when the pipeline is running and see whether that is also picked up rightly. """ prefix = get_random_string(string.ascii_letters, 5) sftp_file_name1 = f'{prefix}{get_random_string(string.ascii_letters, 10)}.txt' sftp_file_name2 = f'{prefix}{get_random_string(string.ascii_letters, 10)}.txt' sftp_file_name3 = f'{prefix}{get_random_string(string.ascii_letters, 10)}.txt' raw_text_data = get_random_string(string.printable, 1000) sftp.put_string(os.path.join(sftp.path, sftp_file_name1), raw_text_data) sftp.chmod(os.path.join(sftp.path, sftp_file_name1), 000) sftp.put_string(os.path.join(sftp.path, sftp_file_name2), raw_text_data) # Build the pipeline builder = sdc_builder.get_pipeline_builder() sftp_ftp_client = builder.add_stage('SFTP/FTP/FTPS Client', type='origin') sftp_ftp_client.set_attributes(file_name_pattern = f'{prefix}*', data_format = 'WHOLE_FILE', batch_wait_time_in_ms = 10000, max_batch_size_in_records = 1) trash = builder.add_stage('Trash') wiretap = builder.add_wiretap() sftp_ftp_client >> [wiretap.destination, trash] sftp_to_trash_pipeline = builder.build().configure_for_environment(sftp) sdc_executor.add_pipeline(sftp_to_trash_pipeline) try: # Wait to capture snapshot till 5 batches start_command = sdc_executor.start_pipeline(sftp_to_trash_pipeline) start_command.wait_for_pipeline_batch_count(2) sftp.put_string(os.path.join(sftp.path, sftp_file_name3), raw_text_data) start_command.wait_for_pipeline_batch_count(5) error_msgs = sdc_executor.get_stage_errors(sftp_to_trash_pipeline, sftp_ftp_client) # Verify the stage error message assert 'REMOTE_DOWNLOAD_10' in [e.error_code for e in error_msgs] actual_records = [record.field['fileInfo']['filename'] for record in wiretap.output_records] sdc_executor.stop_pipeline(sftp_to_trash_pipeline) wiretap.reset() assert [sftp_file_name2, sftp_file_name3] == actual_records finally: # Delete the test SFTP origin files we created transport, sftp_client = sftp.client sftp_client.chmod(os.path.join(sftp.path, sftp_file_name1), 700) try: for sftp_file_name in [sftp_file_name1, sftp_file_name2, sftp_file_name3]: logger.debug('Removing file at %s/%s on SFTP server ...', sftp.path, sftp_file_name) sftp_client.remove(os.path.join(sftp.path, sftp_file_name)) finally: sftp_client.close() transport.close()