def __init__(self, req_id, uri, diss_info, client_ip):
self.req_id = req_id
self.uri = uri
self.diss_info = diss_info
self.date_reception = gmtime()
self.request_file = ""
self._diff_externalid = None
self.hostname = self.get_hostname(client_ip)
#load settings, if it has not been done
if not SettingsManager.is_loaded():
SettingsManager.load_settings()
# initialize LOGGER
setup_logging()
# setup repertory structure
HarnessTree.setup_tree()
# Setting up database if necessary
if Database.get_database() is None:
from webservice.server.application import APP
Database.initialize_database(APP)
LOGGER.debug("Database setup")
LOGGER.debug("Created a Notification object with id %s", req_id)
def setUp(self):
# create repertories and configure harnesss
self.tmpdir = mkdtemp(prefix='harnais_')
os.environ["TMPDIR"] = self.tmpdir
self.staging_post = join(self.tmpdir, "staging_post")
os.mkdir(self.staging_post)
# prepare settings
SettingsManager.load_settings()
SettingsManager.update(dict(harnaisLogdir=self.tmpdir,
harnaisDir=self.tmpdir,
harnaisAckDir=self.tmpdir
), testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(dict(SettingsManager._parameters), file_)
SettingsManager.reset()
# configure SOAP server
self.hostname = hostname = socket.gethostname()
self.port = port = os.environ.get(ENV.port) or PORT
self.soap_url = os.environ[ENV.soap_url]= ('http://{hostname}:{port}/harnais-diss-v2/'
'webservice/Dissemination?wsdl'.format(hostname=hostname,
port=port))
SoapServer.create_server()
# connect to soap WSDL
self.client = Client(self.soap_url)
self.factory = self.client.type_factory('http://dissemination.harness.openwis.org/')
def setUp(self):
# configuring settings and repertories
DebugSettingsManager.debug = "False"
DebugSettingsManager.test_sftp = "True"
self.tmpdir = mkdtemp(prefix='harnais_')
harnais_dir = join(self.tmpdir, "harnais")
os.mkdir(harnais_dir)
os.environ["TMPDIR"] = self.tmpdir
self.staging_post = join(self.tmpdir, "staging_post")
os.mkdir(self.staging_post)
self.difmet_deposit = join(self.tmpdir, "difmet_deposit")
os.mkdir(self.difmet_deposit)
self.ack_dir = join(self.tmpdir, "ack_dir")
os.mkdir(self.ack_dir)
# create files on staging post
for i in range(2):
with open(join(self.staging_post,"A_SNFR30LFPW270700_C_LFPW_20180927070000_%i.txt" % i),"w") as file_out:
file_out.write("Dummy staging post test file")
# configure soap server
self.hostname = hostname = socket.gethostname()
self.port = port = os.environ.get(ENV.port) or PORT
self.soap_url = os.environ[ENV.soap_url]= ('http://{hostname}:{port}/harnais-diss-v2/'
'webservice/Dissemination?wsdl'.format(hostname=hostname,
port=port))
SettingsManager.load_settings()
SettingsManager.update(dict(harnaisLogdir=harnais_dir,
harnaisDir=harnais_dir,
harnaisAckDir=self.ack_dir,
openwisStagingPath=gettempdir(),
openwisHost="localhost",
openwisSftpUser="******",
openwisSftpPassword="******",
openwisSftpPort = 3373,
processFileIdle = 10,
dissHost="0.0." + "0.0",
dissFtpUser="******",
dissFtpPasswd="12345",
dissFtpDir=self.difmet_deposit,
dissFtpMode=None,
dissFtpPort=2121,
sendFTPlimitConn=5),
testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(dict(SettingsManager._parameters), file_)
SettingsManager.reset()
def get_logger():
"""
Fetch logger
"""
SettingsManager.load_settings()
# initialize LOGGER
setup_logging()
logger = logging.getLogger(__name__)
logger.debug("Logging configuration set up for %s", __name__)
return logger
def get(cls, key):
if not SettingsManager.is_loaded():
SettingsManager.load_settings()
LOGGER.info("Settings loaded")
#check if settings have been modified
if SettingsManager.get_checksum() != cls._checksum:
LOGGER.info("Settings have been modified")
cls.setup_tree(update=True)
elif not os.path.isdir(cls._repertories[key]):
cls.setup_tree(update=False)
return cls._repertories[key]
def test_sending(self):
"""
Testing FTP transfer of 5 small files
"""
# prepare settings
SettingsManager.load_settings()
SettingsManager.update(dict(harnaisLogdir=self.tmpdir,
harnaisDir=self.tmpdir,
harnaisAckDir=self.tmpdir,
dissHost="0.0." + "0.0",
dissFtpUser="******",
dissFtpPasswd="12345",
bandwidth=10,
dissFtpDir=self.difmet_deposit,
dissFtpMode=None,
dissFtpPort=2121,
sendFTPlimitConn=5,
sendFTPIdle=10),
testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(SettingsManager._parameters, file_)
setup_logging()
SettingsManager.load_settings()
dir_C = HarnessTree.get("temp_dissRequest_C")
dir_D = HarnessTree.get("temp_dissRequest_D")
#create dummy files of size 1000 bits to send
for i in range(5):
filename = "package_file_%i.tar.gz" % i
with open(join(dir_C, filename), "wb") as file_out:
size = 1000
file_out.seek(size - 1)
file_out.write(b"\0")
DifmetSender.process(max_loops=3)
list_dwld = [
'package_file_0.tar.gz', 'package_file_2.tar.gz',
'package_file_4.tar.gz', 'package_file_1.tar.gz',
'package_file_3.tar.gz'
]
expected_result = True
for file_ in os.listdir(self.difmet_deposit):
expected_result = expected_result and (file_ in list_dwld)
self.assertTrue(expected_result)
def setUp(self):
self.tmpdir = mkdtemp(prefix='harnais_')
harnais_dir = join(self.tmpdir, "harnais")
os.mkdir(harnais_dir)
os.environ["TMPDIR"] = self.tmpdir
self.staging_post = join(self.tmpdir, "staging_post")
os.mkdir(self.staging_post)
self.difmet_deposit = join(self.tmpdir, "difmet_deposit")
os.mkdir(self.difmet_deposit)
self.ack_dir = join(self.tmpdir, "ack_dir")
os.mkdir(self.ack_dir)
self.hostname = hostname = socket.gethostname()
port = os.environ.get(ENV.port) or PORT
os.environ[ENV.soap_url] = ('http://{hostname}:{port}/harnais-diss-v2/'
'webservice/Dissemination?wsdl'.format(hostname=hostname,
port=port))
SettingsManager.load_settings()
SettingsManager.update(dict(harnaisLogdir=harnais_dir,
harnaisDir=harnais_dir,
harnaisAckDir=self.ack_dir,
openwisStagingPath=gettempdir(),
openwisHost="localhost",
openwisSftpUser="******",
openwisSftpPassword="******",
openwisSftpPort = 3373,
processFileIdle = 10,
dissHost="0.0." + "0.0",
dissFtpUser="******",
dissFtpPasswd="12345",
dissFtpDir=self.difmet_deposit,
dissFtpMode=None,
dissFtpPort=2121,
sendFTPlimitConn=5),
testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(SettingsManager._parameters, file_)
setup_logging()
def main():
SettingsManager.load_settings()
# initialize LOGGER
setup_logging()
# setup repertory structure
HarnessTree.setup_tree()
# Setting up database
Database.initialize_database(APP)
LOGGER.info("setup complete")
hostname = socket.gethostname()
port = os.environ.get(ENV.port) or PORT
LOGGER.warning("Starting application through Flask development server."
" This is NOT a production environment.")
LOGGER.info(
"Launching Flask development server "
"on hostname %s on port %s", hostname, port)
APP.run(host=hostname, port=port)
def setUp(self):
self.tmpdir = mkdtemp(prefix='harnais_')
os.environ["TMPDIR"] = self.tmpdir
self.staging_post = join(self.tmpdir, "staging_post")
os.mkdir(self.staging_post)
# # prepare settings
SettingsManager.load_settings()
SettingsManager.update(dict(
harnaisLogdir=self.tmpdir,
harnaisDir=self.tmpdir,
harnaisAckDir=self.tmpdir,
bandwidth=5,
),
testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(SettingsManager._parameters, file_)
setup_logging()
def test_download_large_file(self):
"""
We check that download a big file on staging post is not an issue
"""
self.assertTrue(file_manager.manager.TEST_SFTP)
SettingsManager.reset()
SettingsManager.load_settings()
SettingsManager.update(dict(harnaisLogdir=self.tmpdir,
harnaisDir=self.tmpdir,
harnaisAckDir=self.tmpdir,
openwisStagingPath=gettempdir(),
openwisHost="localhost",
openwisSftpUser="******",
openwisSftpPassword="******",
openwisSftpPort=3373,
bandwidth=5),
testing=True)
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(SettingsManager._parameters, file_)
files_list = []
for i in range(1):
filename = "A_largefile_C_LFPW_20180927070000_%i.txt" % i
files_list.append(filename)
with open(join(self.staging_post, filename), "wb") as file_out:
size = 200 * (1 << 20)
file_out.seek(size - 1)
file_out.write(b"\0")
FileManager.process_instruction_file(self.json_file)
dir_b = HarnessTree.get("temp_dissRequest_B")
for filename in files_list:
self.assertTrue(os.path.isfile(join(dir_b, filename)))
def test_complet(self):
"""
This is a full test. The SOAP server receives 3 requests, 2 mails and 1 ftp
The manager is then started to process the three json files created by the
previous step
When the files are correctly packaed, the sender sends them by FTP
to local deposit. Finally, a ack is simulated and ack_receiver
processes it.
"""
SettingsManager.load_settings()
# start the SOAP server
SoapServer.create_server()
# connect to the WSDL
client = Client(self.soap_url)
factory = client.type_factory('http://dissemination.harness.openwis.org/')
# create a dummy request MailDiffusion
test_diffusion = factory.MailDiffusion(address="*****@*****.**",
headerLine="dummyHeaderLine",
subject= "dummySubject",
dispatchMode = "TO",
attachmentMode="AS_ATTACHMENT")
info = factory.DisseminationInfo(priority=5,SLA=6,dataPolicy="dummyDataPolicy", diffusion=test_diffusion)
# send the mail requests
result1 = client.service.disseminate(requestId="123456", fileURI=self.staging_post, disseminationInfo=info)
result2 = client.service.disseminate(requestId="654321", fileURI=self.staging_post, disseminationInfo=info)
# create a dummy request FTPDiffusion
test_diffusion = factory.FTPDiffusion(host="dummyHost",
port="dummyPort",
user="******",
password="******",
passive="False",
remotePath="dummyPath",
checkFileSize="True",
encrypted="False")
info = factory.DisseminationInfo(priority=5,SLA=6,
dataPolicy="dummyDataPolicy",
diffusion=test_diffusion)
# send the ftp requests
result3 = client.service.disseminate(requestId="111111", fileURI=self.staging_post, disseminationInfo=info)
print(result1)
print(result2)
print(result3)
#stop the SOAP
SoapServer.stop_server()
# create the SFTP server
SFTPserver.create_server(self.staging_post)
# SettingsManager.update(dict(openwisStagingPath=gettempdir(),
# openwisHost="localhost",
# openwisSftpUser="******",
# openwisSftpPassword="******",
# openwisSftpPort = 3373
# ),
# testing=True)
# Start the manager for 1 loop
thr = Thread(target=FileManager.process, kwargs={"max_loops":1})
try:
thr.start()
thr.join()
# STOP sftp server when the FileManager is done
# processing
SFTPserver.stop_server()
print("Manager finished")
except KeyboardInterrupt:
SFTPserver.stop_server()
sleep(10)
# create the ftp server
Tools.kill_process("diffmet_test_ftp_server")
FTPserver.create_server("/")
# send the files
thr = Thread(target=DifmetSender.process, kwargs={"max_loops":3})
try:
thr.start()
thr.join()
print("DifMet finished")
# STOP sftp server when the DifmetSender is done
# processing
FTPserver.stop_server()
except KeyboardInterrupt:
FTPserver.stop_server()
# get the diff_external_id in the database required for to simulate an ack
with Database.get_app().app_context():
records = Diffusion.query.filter(Diffusion.fullrequestId.contains("123456")).all()
print(records[0].fullrequestId)
ext_id1 = records[0].diff_externalid
ext_id2 = records[1].diff_externalid
#simulate an ack difmet file
with open(join(self.ack_dir, "ack_file.acqdifmet.xml"),"w") as file_:
file_.write("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n"
"<acquittements>\n"
"<acquittement>\n"
"<date>2018-10-01T12:31:46Z</date>\n"
"<type>RECEIVED</type>\n"
"<status>OK</status>\n"
"<productid>fr-met,SNFR30LFPW011000LFPW,00000-wiss,20181001100000</productid>\n"
"<product_internalid>66180_20181001123146</product_internalid>\n"
"<send2>0</send2>\n"
"<diffusion_externalid>{ext_id1}</diffusion_externalid>\n"
"<diffusion_internalid>66181_20181001123146</diffusion_internalid>\n"
"<channel>EMAIL</channel>\n"
"<media>EMAIL</media>\n"
"<use_standby>0</use_standby>\n"
"<email_adress>[email protected]</email_adress>\n"
"</acquittement>\n"
"<acquittement>\n"
"<date>2018-10-01T12:31:46Z</date>\n"
"<type>SEND</type>\n"
"<status>OK</status>\n"
"<productid>fr-met,SNFR30LFPW011000LFPW,00001-wiss,20181001100000</productid>\n"
"<product_internalid>66180_20181001123146</product_internalid>\n"
"<send2>0</send2>\n"
"<diffusion_externalid>{ext_id2}</diffusion_externalid>\n"
"<diffusion_internalid>66181_20181001123146</diffusion_internalid>\n"
"<channel>EMAIL</channel>\n"
"<media>EMAIL</media>\n"
"<use_standby>0</use_standby>\n"
"<try_number>1</try_number>\n"
"<email_adress>[email protected]</email_adress>\n"
"<comment>nom de fichier en attachement au courriel: machin</comment>\n"
"</acquittement>\n"
"<acquittement>\n"
"<date>2018-10-01T12:31:46Z</date>\n"
"<type>SEND</type>\n"
"<status>OK</status>\n"
"<productid>fr-met,SNFR30LFPW011000LFPW,00000-wiss,20181001100000</productid>\n"
"<product_internalid>66180_20181001123146</product_internalid>\n"
"<send2>0</send2>\n"
"<diffusion_externalid>{ext_id1}</diffusion_externalid>\n"
"<diffusion_internalid>66181_20181001123146</diffusion_internalid>\n"
"<channel>EMAIL</channel>\n"
"<media>EMAIL</media>\n"
"<use_standby>0</use_standby>\n"
"<try_number>1</try_number>\n"
"<email_adress>[email protected]</email_adress>\n"
"<comment>nom de fichier en attachement au courriel: machin</comment>\n"
"</acquittement>\n"
"<acquittementnumber>3</acquittementnumber>\n"
"</acquittements>".format(ext_id1=ext_id1, ext_id2=ext_id2))
# start the ack_receiver to process the ack file generated
thr = Thread(target=AckReceiver.process, kwargs={"max_loops":2})
thr.start()
thr.join()
print("Ack_receiver finished")
# check acquittement
SoapServer.create_server()
client = Client(self.soap_url)
factory = client.type_factory('http://dissemination.harness.openwis.org/')
result = client.service.monitorDissemination(requestId="123456")
print(result)
result = client.service.monitorDissemination(requestId="654321")
print(result)
SoapServer.stop_server()
error_log = join(self.tmpdir, "harnais/errors.log")
# if no errors, test is successful.
with open(error_log, "r") as file_:
self.assertEqual(file_.read(),"")
def setUp(self):
# Configuring repertories
file_manager.manager.TEST_SFTP = True
self.tmpdir = mkdtemp(prefix='harnais_')
os.environ["TMPDIR"] = self.tmpdir
self.staging_post = join(self.tmpdir, "staging_post")
os.mkdir(self.staging_post)
# # prepare settings
SettingsManager.load_settings()
SettingsManager.update(dict(harnaisLogdir=self.tmpdir,
harnaisDir=self.tmpdir,
harnaisAckDir=self.tmpdir,
openwisStagingPath=gettempdir(),
openwisHost="localhost",
openwisSftpUser="******",
openwisSftpPassword="******",
openwisSftpPort=3373),
testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(SettingsManager._parameters, file_)
setup_logging()
# Start sftp server
SFTPserver.create_server(self.staging_post)
# create json file to process
self.dir_a = HarnessTree.get("temp_dissRequest_A")
self.json_file = json_file = join(self.dir_a,
"test_instruction_file.json")
instr = {
'hostname': socket.gethostname(),
'uri': self.staging_post,
'req_id': '123456',
'diffpriority': 81,
'date': datetime.now().strftime("%Y%m%d%H%M%S"),
'diffusion': {
'fileName': None,
'attachmentMode': 'AS_ATTACHMENT',
'dispatchMode': 'TO',
'DiffusionType': 'EMAIL',
'subject': 'dummySubject',
'headerLine': 'dummyHeaderLine',
'address': '*****@*****.**'
}
}
# put it in cache/A_dissreq
with open(json_file, "w") as file_:
json.dump(instr, file_)
# create corresponding record in database:
ext_id = Tools.generate_random_string()
diffusion = Diffusion(diff_externalid=ext_id,
fullrequestId="123456" + socket.gethostname(),
requestStatus=REQ_STATUS.ongoing,
Date=datetime.now(),
rxnotif=True,
message="Created record in SQL database")
with Database.get_app().app_context():
database = Database.get_database()
database.session.add(diffusion)
database.session.commit()
def load_settings():
loaded = SettingsManager.load_settings()
if loaded:
LOGGER.debug("Settings loaded")
if __name__ == '__main__':
# this is used for testing and debugging purpose only. It allows to launch the process independently
# for user specified n loops
process_name = "harness_file_manager"
setproctitle(process_name)
parser = argparse.ArgumentParser(description='File Manager process loop.')
parser.add_argument("--loops", help=("How many loops should the process "
"execute.\n "
"Loop indefinitly if no value is "
"provided"),
type=int,
nargs=1)
args = parser.parse_args()
if args.loops:
max_loops = args.loops[0]
else:
max_loops = 0
# initialize LOGGER
SettingsManager.load_settings()
setup_logging()
LOGGER = logging.getLogger("file_manager.manager")
LOGGER.debug("Logging configuration set up for %s", "file_manager.manager")
LOGGER.info("File Manager setup complete")
FileManager.process(max_loops)