def process(self):
# create the unicity key for the database
self._diff_externalid = diff_id = Tools.generate_random_string()
# fetch database
database = Database.get_database()
try:
# create JSON request file
self.create_request_file()
# create the first database record
diffusion = Diffusion(diff_externalid=diff_id,
fullrequestId=self.req_id+self.hostname,
requestStatus=REQ_STATUS.ongoing,
Date=self._to_datetime(self.date_reception),
rxnotif=True,
message="Created record in SQL database",)
with Database.get_app().app_context():
database.session.add(diffusion)
database.session.commit()
LOGGER.debug("Committed %s dissemination status "
"into database.", REQ_STATUS.ongoing)
status = REQ_STATUS.ongoing
except Exception as exc:
LOGGER.exception("Error during notification processing. "
"Dissemination failed.")
status = self.commit_failure(database, diff_id)
return status
def update_filename(self, filename):
"""
Register a new entry in the database in case
there are multiples files for one request_id. If one request id
is requesting N files, there should be N entries in the database, each with its own diff_external_id.
"""
# fetch database
database = Database.get_database()
# get the base record to duplicate
with Database.get_app().app_context():
base_record = Diffusion.query.filter_by(
fullrequestId=self.req_id).first()
if base_record.original_file is None:
# if there is only the one record created by
# receiver module, job is done
base_record.original_file = filename
database.session.commit()
# otherwise, we create a new record
else:
diffusion = Diffusion(diff_externalid=Tools.generate_random_string(),
fullrequestId=base_record.fullrequestId,
original_file=filename,
requestStatus=base_record.requestStatus,
message=base_record.message,
Date=base_record.Date,
rxnotif=base_record.rxnotif)
database.session.add(diffusion)
database.session.commit()
def commit_failure(self, database, diff_id):
diffusion = Diffusion(diff_externalid=diff_id,
fullrequestId=self.req_id,
requestStatus=REQ_STATUS.failed,
Date=self._to_datetime(self.date_reception),
rxnotif=True)
with Database.get_app().app_context():
database.session.add(diffusion)
database.session.commit()
LOGGER.info("Committed %s dissemination status into database.",
REQ_STATUS.failed)
if os.path.isfile(self.request_file):
Tools.remove_file(self.request_file, "JSON request", LOGGER)
return REQ_STATUS.failed
def test_complet(self):
"""
This is a full test. The SOAP server receives 3 requests, 2 mails and 1 ftp
The manager is then started to process the three json files created by the
previous step
When the files are correctly packaed, the sender sends them by FTP
to local deposit. Finally, a ack is simulated and ack_receiver
processes it.
"""
SettingsManager.load_settings()
# start the SOAP server
SoapServer.create_server()
# connect to the WSDL
client = Client(self.soap_url)
factory = client.type_factory('http://dissemination.harness.openwis.org/')
# create a dummy request MailDiffusion
test_diffusion = factory.MailDiffusion(address="*****@*****.**",
headerLine="dummyHeaderLine",
subject= "dummySubject",
dispatchMode = "TO",
attachmentMode="AS_ATTACHMENT")
info = factory.DisseminationInfo(priority=5,SLA=6,dataPolicy="dummyDataPolicy", diffusion=test_diffusion)
# send the mail requests
result1 = client.service.disseminate(requestId="123456", fileURI=self.staging_post, disseminationInfo=info)
result2 = client.service.disseminate(requestId="654321", fileURI=self.staging_post, disseminationInfo=info)
# create a dummy request FTPDiffusion
test_diffusion = factory.FTPDiffusion(host="dummyHost",
port="dummyPort",
user="******",
password="******",
passive="False",
remotePath="dummyPath",
checkFileSize="True",
encrypted="False")
info = factory.DisseminationInfo(priority=5,SLA=6,
dataPolicy="dummyDataPolicy",
diffusion=test_diffusion)
# send the ftp requests
result3 = client.service.disseminate(requestId="111111", fileURI=self.staging_post, disseminationInfo=info)
print(result1)
print(result2)
print(result3)
#stop the SOAP
SoapServer.stop_server()
# create the SFTP server
SFTPserver.create_server(self.staging_post)
# SettingsManager.update(dict(openwisStagingPath=gettempdir(),
# openwisHost="localhost",
# openwisSftpUser="******",
# openwisSftpPassword="******",
# openwisSftpPort = 3373
# ),
# testing=True)
# Start the manager for 1 loop
thr = Thread(target=FileManager.process, kwargs={"max_loops":1})
try:
thr.start()
thr.join()
# STOP sftp server when the FileManager is done
# processing
SFTPserver.stop_server()
print("Manager finished")
except KeyboardInterrupt:
SFTPserver.stop_server()
sleep(10)
# create the ftp server
Tools.kill_process("diffmet_test_ftp_server")
FTPserver.create_server("/")
# send the files
thr = Thread(target=DifmetSender.process, kwargs={"max_loops":3})
try:
thr.start()
thr.join()
print("DifMet finished")
# STOP sftp server when the DifmetSender is done
# processing
FTPserver.stop_server()
except KeyboardInterrupt:
FTPserver.stop_server()
# get the diff_external_id in the database required for to simulate an ack
with Database.get_app().app_context():
records = Diffusion.query.filter(Diffusion.fullrequestId.contains("123456")).all()
print(records[0].fullrequestId)
ext_id1 = records[0].diff_externalid
ext_id2 = records[1].diff_externalid
#simulate an ack difmet file
with open(join(self.ack_dir, "ack_file.acqdifmet.xml"),"w") as file_:
file_.write("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n"
"<acquittements>\n"
"<acquittement>\n"
"<date>2018-10-01T12:31:46Z</date>\n"
"<type>RECEIVED</type>\n"
"<status>OK</status>\n"
"<productid>fr-met,SNFR30LFPW011000LFPW,00000-wiss,20181001100000</productid>\n"
"<product_internalid>66180_20181001123146</product_internalid>\n"
"<send2>0</send2>\n"
"<diffusion_externalid>{ext_id1}</diffusion_externalid>\n"
"<diffusion_internalid>66181_20181001123146</diffusion_internalid>\n"
"<channel>EMAIL</channel>\n"
"<media>EMAIL</media>\n"
"<use_standby>0</use_standby>\n"
"<email_adress>[email protected]</email_adress>\n"
"</acquittement>\n"
"<acquittement>\n"
"<date>2018-10-01T12:31:46Z</date>\n"
"<type>SEND</type>\n"
"<status>OK</status>\n"
"<productid>fr-met,SNFR30LFPW011000LFPW,00001-wiss,20181001100000</productid>\n"
"<product_internalid>66180_20181001123146</product_internalid>\n"
"<send2>0</send2>\n"
"<diffusion_externalid>{ext_id2}</diffusion_externalid>\n"
"<diffusion_internalid>66181_20181001123146</diffusion_internalid>\n"
"<channel>EMAIL</channel>\n"
"<media>EMAIL</media>\n"
"<use_standby>0</use_standby>\n"
"<try_number>1</try_number>\n"
"<email_adress>[email protected]</email_adress>\n"
"<comment>nom de fichier en attachement au courriel: machin</comment>\n"
"</acquittement>\n"
"<acquittement>\n"
"<date>2018-10-01T12:31:46Z</date>\n"
"<type>SEND</type>\n"
"<status>OK</status>\n"
"<productid>fr-met,SNFR30LFPW011000LFPW,00000-wiss,20181001100000</productid>\n"
"<product_internalid>66180_20181001123146</product_internalid>\n"
"<send2>0</send2>\n"
"<diffusion_externalid>{ext_id1}</diffusion_externalid>\n"
"<diffusion_internalid>66181_20181001123146</diffusion_internalid>\n"
"<channel>EMAIL</channel>\n"
"<media>EMAIL</media>\n"
"<use_standby>0</use_standby>\n"
"<try_number>1</try_number>\n"
"<email_adress>[email protected]</email_adress>\n"
"<comment>nom de fichier en attachement au courriel: machin</comment>\n"
"</acquittement>\n"
"<acquittementnumber>3</acquittementnumber>\n"
"</acquittements>".format(ext_id1=ext_id1, ext_id2=ext_id2))
# start the ack_receiver to process the ack file generated
thr = Thread(target=AckReceiver.process, kwargs={"max_loops":2})
thr.start()
thr.join()
print("Ack_receiver finished")
# check acquittement
SoapServer.create_server()
client = Client(self.soap_url)
factory = client.type_factory('http://dissemination.harness.openwis.org/')
result = client.service.monitorDissemination(requestId="123456")
print(result)
result = client.service.monitorDissemination(requestId="654321")
print(result)
SoapServer.stop_server()
error_log = join(self.tmpdir, "harnais/errors.log")
# if no errors, test is successful.
with open(error_log, "r") as file_:
self.assertEqual(file_.read(),"")
def setUp(self):
# Configuring repertories
file_manager.manager.TEST_SFTP = True
self.tmpdir = mkdtemp(prefix='harnais_')
os.environ["TMPDIR"] = self.tmpdir
self.staging_post = join(self.tmpdir, "staging_post")
os.mkdir(self.staging_post)
# # prepare settings
SettingsManager.load_settings()
SettingsManager.update(dict(harnaisLogdir=self.tmpdir,
harnaisDir=self.tmpdir,
harnaisAckDir=self.tmpdir,
openwisStagingPath=gettempdir(),
openwisHost="localhost",
openwisSftpUser="******",
openwisSftpPassword="******",
openwisSftpPort=3373),
testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(SettingsManager._parameters, file_)
setup_logging()
# Start sftp server
SFTPserver.create_server(self.staging_post)
# create json file to process
self.dir_a = HarnessTree.get("temp_dissRequest_A")
self.json_file = json_file = join(self.dir_a,
"test_instruction_file.json")
instr = {
'hostname': socket.gethostname(),
'uri': self.staging_post,
'req_id': '123456',
'diffpriority': 81,
'date': datetime.now().strftime("%Y%m%d%H%M%S"),
'diffusion': {
'fileName': None,
'attachmentMode': 'AS_ATTACHMENT',
'dispatchMode': 'TO',
'DiffusionType': 'EMAIL',
'subject': 'dummySubject',
'headerLine': 'dummyHeaderLine',
'address': '*****@*****.**'
}
}
# put it in cache/A_dissreq
with open(json_file, "w") as file_:
json.dump(instr, file_)
# create corresponding record in database:
ext_id = Tools.generate_random_string()
diffusion = Diffusion(diff_externalid=ext_id,
fullrequestId="123456" + socket.gethostname(),
requestStatus=REQ_STATUS.ongoing,
Date=datetime.now(),
rxnotif=True,
message="Created record in SQL database")
with Database.get_app().app_context():
database = Database.get_database()
database.session.add(diffusion)
database.session.commit()
