def __init__(self, req_id, uri, diss_info, client_ip):
self.req_id = req_id
self.uri = uri
self.diss_info = diss_info
self.date_reception = gmtime()
self.request_file = ""
self._diff_externalid = None
self.hostname = self.get_hostname(client_ip)
#load settings, if it has not been done
if not SettingsManager.is_loaded():
SettingsManager.load_settings()
# initialize LOGGER
setup_logging()
# setup repertory structure
HarnessTree.setup_tree()
# Setting up database if necessary
if Database.get_database() is None:
from webservice.server.application import APP
Database.initialize_database(APP)
LOGGER.debug("Database setup")
LOGGER.debug("Created a Notification object with id %s", req_id)
def setup_process(cls):
if not cls._running:
setup_logging()
LOGGER = logging.getLogger(__name__)
LOGGER.info("Sender process starting")
# create tree structure if necessary
HarnessTree.setup_tree()
cls._running = True
def get_logger():
"""
Fetch logger
"""
SettingsManager.load_settings()
# initialize LOGGER
setup_logging()
logger = logging.getLogger(__name__)
logger.debug("Logging configuration set up for %s", __name__)
return logger
def test_sending(self):
"""
Testing FTP transfer of 5 small files
"""
# prepare settings
SettingsManager.load_settings()
SettingsManager.update(dict(harnaisLogdir=self.tmpdir,
harnaisDir=self.tmpdir,
harnaisAckDir=self.tmpdir,
dissHost="0.0." + "0.0",
dissFtpUser="******",
dissFtpPasswd="12345",
bandwidth=10,
dissFtpDir=self.difmet_deposit,
dissFtpMode=None,
dissFtpPort=2121,
sendFTPlimitConn=5,
sendFTPIdle=10),
testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(SettingsManager._parameters, file_)
setup_logging()
SettingsManager.load_settings()
dir_C = HarnessTree.get("temp_dissRequest_C")
dir_D = HarnessTree.get("temp_dissRequest_D")
#create dummy files of size 1000 bits to send
for i in range(5):
filename = "package_file_%i.tar.gz" % i
with open(join(dir_C, filename), "wb") as file_out:
size = 1000
file_out.seek(size - 1)
file_out.write(b"\0")
DifmetSender.process(max_loops=3)
list_dwld = [
'package_file_0.tar.gz', 'package_file_2.tar.gz',
'package_file_4.tar.gz', 'package_file_1.tar.gz',
'package_file_3.tar.gz'
]
expected_result = True
for file_ in os.listdir(self.difmet_deposit):
expected_result = expected_result and (file_ in list_dwld)
self.assertTrue(expected_result)
def setup_process(cls):
"""
Load settings, initialize logging and tree structure
and connect to database
"""
if not cls._running:
setup_logging()
LOGGER = logging.getLogger(__name__)
LOGGER.info("File manager process starting")
# create tree structure if necessary
HarnessTree.setup_tree()
#connect the database
Database.initialize_database(APP)
cls._running = True
def setUp(self):
self.tmpdir = mkdtemp(prefix='harnais_')
harnais_dir = join(self.tmpdir, "harnais")
os.mkdir(harnais_dir)
os.environ["TMPDIR"] = self.tmpdir
self.staging_post = join(self.tmpdir, "staging_post")
os.mkdir(self.staging_post)
self.difmet_deposit = join(self.tmpdir, "difmet_deposit")
os.mkdir(self.difmet_deposit)
self.ack_dir = join(self.tmpdir, "ack_dir")
os.mkdir(self.ack_dir)
self.hostname = hostname = socket.gethostname()
port = os.environ.get(ENV.port) or PORT
os.environ[ENV.soap_url] = ('http://{hostname}:{port}/harnais-diss-v2/'
'webservice/Dissemination?wsdl'.format(hostname=hostname,
port=port))
SettingsManager.load_settings()
SettingsManager.update(dict(harnaisLogdir=harnais_dir,
harnaisDir=harnais_dir,
harnaisAckDir=self.ack_dir,
openwisStagingPath=gettempdir(),
openwisHost="localhost",
openwisSftpUser="******",
openwisSftpPassword="******",
openwisSftpPort = 3373,
processFileIdle = 10,
dissHost="0.0." + "0.0",
dissFtpUser="******",
dissFtpPasswd="12345",
dissFtpDir=self.difmet_deposit,
dissFtpMode=None,
dissFtpPort=2121,
sendFTPlimitConn=5),
testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(SettingsManager._parameters, file_)
setup_logging()
def main():
SettingsManager.load_settings()
# initialize LOGGER
setup_logging()
# setup repertory structure
HarnessTree.setup_tree()
# Setting up database
Database.initialize_database(APP)
LOGGER.info("setup complete")
hostname = socket.gethostname()
port = os.environ.get(ENV.port) or PORT
LOGGER.warning("Starting application through Flask development server."
" This is NOT a production environment.")
LOGGER.info(
"Launching Flask development server "
"on hostname %s on port %s", hostname, port)
APP.run(host=hostname, port=port)
def setUp(self):
self.tmpdir = mkdtemp(prefix='harnais_')
os.environ["TMPDIR"] = self.tmpdir
self.staging_post = join(self.tmpdir, "staging_post")
os.mkdir(self.staging_post)
# # prepare settings
SettingsManager.load_settings()
SettingsManager.update(dict(
harnaisLogdir=self.tmpdir,
harnaisDir=self.tmpdir,
harnaisAckDir=self.tmpdir,
bandwidth=5,
),
testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(SettingsManager._parameters, file_)
setup_logging()
def setUp(self):
# Configuring repertories
file_manager.manager.TEST_SFTP = True
self.tmpdir = mkdtemp(prefix='harnais_')
os.environ["TMPDIR"] = self.tmpdir
self.staging_post = join(self.tmpdir, "staging_post")
os.mkdir(self.staging_post)
# # prepare settings
SettingsManager.load_settings()
SettingsManager.update(dict(harnaisLogdir=self.tmpdir,
harnaisDir=self.tmpdir,
harnaisAckDir=self.tmpdir,
openwisStagingPath=gettempdir(),
openwisHost="localhost",
openwisSftpUser="******",
openwisSftpPassword="******",
openwisSftpPort=3373),
testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(SettingsManager._parameters, file_)
setup_logging()
# Start sftp server
SFTPserver.create_server(self.staging_post)
# create json file to process
self.dir_a = HarnessTree.get("temp_dissRequest_A")
self.json_file = json_file = join(self.dir_a,
"test_instruction_file.json")
instr = {
'hostname': socket.gethostname(),
'uri': self.staging_post,
'req_id': '123456',
'diffpriority': 81,
'date': datetime.now().strftime("%Y%m%d%H%M%S"),
'diffusion': {
'fileName': None,
'attachmentMode': 'AS_ATTACHMENT',
'dispatchMode': 'TO',
'DiffusionType': 'EMAIL',
'subject': 'dummySubject',
'headerLine': 'dummyHeaderLine',
'address': '*****@*****.**'
}
}
# put it in cache/A_dissreq
with open(json_file, "w") as file_:
json.dump(instr, file_)
# create corresponding record in database:
ext_id = Tools.generate_random_string()
diffusion = Diffusion(diff_externalid=ext_id,
fullrequestId="123456" + socket.gethostname(),
requestStatus=REQ_STATUS.ongoing,
Date=datetime.now(),
rxnotif=True,
message="Created record in SQL database")
with Database.get_app().app_context():
database = Database.get_database()
database.session.add(diffusion)
database.session.commit()
if __name__ == '__main__':
# this is used for testing and debugging purpose only. It allows to launch the process independently
# for user specified n loops
process_name = "harness_file_manager"
setproctitle(process_name)
parser = argparse.ArgumentParser(description='File Manager process loop.')
parser.add_argument("--loops", help=("How many loops should the process "
"execute.\n "
"Loop indefinitly if no value is "
"provided"),
type=int,
nargs=1)
args = parser.parse_args()
if args.loops:
max_loops = args.loops[0]
else:
max_loops = 0
# initialize LOGGER
SettingsManager.load_settings()
setup_logging()
LOGGER = logging.getLogger("file_manager.manager")
LOGGER.debug("Logging configuration set up for %s", "file_manager.manager")
LOGGER.info("File Manager setup complete")
FileManager.process(max_loops)