def setUp(self):
# create repertories and configure harnesss
self.tmpdir = mkdtemp(prefix='harnais_')
os.environ["TMPDIR"] = self.tmpdir
self.staging_post = join(self.tmpdir, "staging_post")
os.mkdir(self.staging_post)
# prepare settings
SettingsManager.load_settings()
SettingsManager.update(dict(harnaisLogdir=self.tmpdir,
harnaisDir=self.tmpdir,
harnaisAckDir=self.tmpdir
), testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(dict(SettingsManager._parameters), file_)
SettingsManager.reset()
# configure SOAP server
self.hostname = hostname = socket.gethostname()
self.port = port = os.environ.get(ENV.port) or PORT
self.soap_url = os.environ[ENV.soap_url]= ('http://{hostname}:{port}/harnais-diss-v2/'
'webservice/Dissemination?wsdl'.format(hostname=hostname,
port=port))
SoapServer.create_server()
# connect to soap WSDL
self.client = Client(self.soap_url)
self.factory = self.client.type_factory('http://dissemination.harness.openwis.org/')
def setUp(self):
# configuring settings and repertories
DebugSettingsManager.debug = "False"
DebugSettingsManager.test_sftp = "True"
self.tmpdir = mkdtemp(prefix='harnais_')
harnais_dir = join(self.tmpdir, "harnais")
os.mkdir(harnais_dir)
os.environ["TMPDIR"] = self.tmpdir
self.staging_post = join(self.tmpdir, "staging_post")
os.mkdir(self.staging_post)
self.difmet_deposit = join(self.tmpdir, "difmet_deposit")
os.mkdir(self.difmet_deposit)
self.ack_dir = join(self.tmpdir, "ack_dir")
os.mkdir(self.ack_dir)
# create files on staging post
for i in range(2):
with open(join(self.staging_post,"A_SNFR30LFPW270700_C_LFPW_20180927070000_%i.txt" % i),"w") as file_out:
file_out.write("Dummy staging post test file")
# configure soap server
self.hostname = hostname = socket.gethostname()
self.port = port = os.environ.get(ENV.port) or PORT
self.soap_url = os.environ[ENV.soap_url]= ('http://{hostname}:{port}/harnais-diss-v2/'
'webservice/Dissemination?wsdl'.format(hostname=hostname,
port=port))
SettingsManager.load_settings()
SettingsManager.update(dict(harnaisLogdir=harnais_dir,
harnaisDir=harnais_dir,
harnaisAckDir=self.ack_dir,
openwisStagingPath=gettempdir(),
openwisHost="localhost",
openwisSftpUser="******",
openwisSftpPassword="******",
openwisSftpPort = 3373,
processFileIdle = 10,
dissHost="0.0." + "0.0",
dissFtpUser="******",
dissFtpPasswd="12345",
dissFtpDir=self.difmet_deposit,
dissFtpMode=None,
dissFtpPort=2121,
sendFTPlimitConn=5),
testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(dict(SettingsManager._parameters), file_)
SettingsManager.reset()
def test_sending(self):
"""
Testing FTP transfer of 5 small files
"""
# prepare settings
SettingsManager.load_settings()
SettingsManager.update(dict(harnaisLogdir=self.tmpdir,
harnaisDir=self.tmpdir,
harnaisAckDir=self.tmpdir,
dissHost="0.0." + "0.0",
dissFtpUser="******",
dissFtpPasswd="12345",
bandwidth=10,
dissFtpDir=self.difmet_deposit,
dissFtpMode=None,
dissFtpPort=2121,
sendFTPlimitConn=5,
sendFTPIdle=10),
testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(SettingsManager._parameters, file_)
setup_logging()
SettingsManager.load_settings()
dir_C = HarnessTree.get("temp_dissRequest_C")
dir_D = HarnessTree.get("temp_dissRequest_D")
#create dummy files of size 1000 bits to send
for i in range(5):
filename = "package_file_%i.tar.gz" % i
with open(join(dir_C, filename), "wb") as file_out:
size = 1000
file_out.seek(size - 1)
file_out.write(b"\0")
DifmetSender.process(max_loops=3)
list_dwld = [
'package_file_0.tar.gz', 'package_file_2.tar.gz',
'package_file_4.tar.gz', 'package_file_1.tar.gz',
'package_file_3.tar.gz'
]
expected_result = True
for file_ in os.listdir(self.difmet_deposit):
expected_result = expected_result and (file_ in list_dwld)
self.assertTrue(expected_result)
def setUp(self):
self.tmpdir = mkdtemp(prefix='harnais_')
harnais_dir = join(self.tmpdir, "harnais")
os.mkdir(harnais_dir)
os.environ["TMPDIR"] = self.tmpdir
self.staging_post = join(self.tmpdir, "staging_post")
os.mkdir(self.staging_post)
self.difmet_deposit = join(self.tmpdir, "difmet_deposit")
os.mkdir(self.difmet_deposit)
self.ack_dir = join(self.tmpdir, "ack_dir")
os.mkdir(self.ack_dir)
self.hostname = hostname = socket.gethostname()
port = os.environ.get(ENV.port) or PORT
os.environ[ENV.soap_url] = ('http://{hostname}:{port}/harnais-diss-v2/'
'webservice/Dissemination?wsdl'.format(hostname=hostname,
port=port))
SettingsManager.load_settings()
SettingsManager.update(dict(harnaisLogdir=harnais_dir,
harnaisDir=harnais_dir,
harnaisAckDir=self.ack_dir,
openwisStagingPath=gettempdir(),
openwisHost="localhost",
openwisSftpUser="******",
openwisSftpPassword="******",
openwisSftpPort = 3373,
processFileIdle = 10,
dissHost="0.0." + "0.0",
dissFtpUser="******",
dissFtpPasswd="12345",
dissFtpDir=self.difmet_deposit,
dissFtpMode=None,
dissFtpPort=2121,
sendFTPlimitConn=5),
testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(SettingsManager._parameters, file_)
setup_logging()
def setUp(self):
self.tmpdir = mkdtemp(prefix='harnais_')
os.environ["TMPDIR"] = self.tmpdir
self.staging_post = join(self.tmpdir, "staging_post")
os.mkdir(self.staging_post)
# # prepare settings
SettingsManager.load_settings()
SettingsManager.update(dict(
harnaisLogdir=self.tmpdir,
harnaisDir=self.tmpdir,
harnaisAckDir=self.tmpdir,
bandwidth=5,
),
testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(SettingsManager._parameters, file_)
setup_logging()
def test_download_large_file(self):
"""
We check that download a big file on staging post is not an issue
"""
self.assertTrue(file_manager.manager.TEST_SFTP)
SettingsManager.reset()
SettingsManager.load_settings()
SettingsManager.update(dict(harnaisLogdir=self.tmpdir,
harnaisDir=self.tmpdir,
harnaisAckDir=self.tmpdir,
openwisStagingPath=gettempdir(),
openwisHost="localhost",
openwisSftpUser="******",
openwisSftpPassword="******",
openwisSftpPort=3373,
bandwidth=5),
testing=True)
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(SettingsManager._parameters, file_)
files_list = []
for i in range(1):
filename = "A_largefile_C_LFPW_20180927070000_%i.txt" % i
files_list.append(filename)
with open(join(self.staging_post, filename), "wb") as file_out:
size = 200 * (1 << 20)
file_out.seek(size - 1)
file_out.write(b"\0")
FileManager.process_instruction_file(self.json_file)
dir_b = HarnessTree.get("temp_dissRequest_B")
for filename in files_list:
self.assertTrue(os.path.isfile(join(dir_b, filename)))
def setUp(self):
# Configuring repertories
file_manager.manager.TEST_SFTP = True
self.tmpdir = mkdtemp(prefix='harnais_')
os.environ["TMPDIR"] = self.tmpdir
self.staging_post = join(self.tmpdir, "staging_post")
os.mkdir(self.staging_post)
# # prepare settings
SettingsManager.load_settings()
SettingsManager.update(dict(harnaisLogdir=self.tmpdir,
harnaisDir=self.tmpdir,
harnaisAckDir=self.tmpdir,
openwisStagingPath=gettempdir(),
openwisHost="localhost",
openwisSftpUser="******",
openwisSftpPassword="******",
openwisSftpPort=3373),
testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(SettingsManager._parameters, file_)
setup_logging()
# Start sftp server
SFTPserver.create_server(self.staging_post)
# create json file to process
self.dir_a = HarnessTree.get("temp_dissRequest_A")
self.json_file = json_file = join(self.dir_a,
"test_instruction_file.json")
instr = {
'hostname': socket.gethostname(),
'uri': self.staging_post,
'req_id': '123456',
'diffpriority': 81,
'date': datetime.now().strftime("%Y%m%d%H%M%S"),
'diffusion': {
'fileName': None,
'attachmentMode': 'AS_ATTACHMENT',
'dispatchMode': 'TO',
'DiffusionType': 'EMAIL',
'subject': 'dummySubject',
'headerLine': 'dummyHeaderLine',
'address': '*****@*****.**'
}
}
# put it in cache/A_dissreq
with open(json_file, "w") as file_:
json.dump(instr, file_)
# create corresponding record in database:
ext_id = Tools.generate_random_string()
diffusion = Diffusion(diff_externalid=ext_id,
fullrequestId="123456" + socket.gethostname(),
requestStatus=REQ_STATUS.ongoing,
Date=datetime.now(),
rxnotif=True,
message="Created record in SQL database")
with Database.get_app().app_context():
database = Database.get_database()
database.session.add(diffusion)
database.session.commit()