def store_params(param_data, config, eventsource, eventsourcecode):
"""Store parametric data in ComCat.
Args:
config (dict): Dictionary containing fields:
- java Location of Java binary.
- jarfile Location of PDL jar file.
- privatekey Location of PDL private key.
- configfile Location of PDL config file.
- product_source Network contributing this product to ComCat.
eventsource (str): Network that originated the event.
eventsourcecode (str): Event code from network that originated
the event.
Returns:
int: Number of files transferred (should always be 1)
str: Message with any error information.
"""
props = {}
props.update(config)
props['source'] = props['product_source']
del props['product_source']
props['eventsource'] = eventsource
props['eventsourcecode'] = eventsourcecode
props['code'] = eventsource + eventsourcecode
props['type'] = PRODUCT_TYPE
tdir = tempfile.mkdtemp()
jsonfile = os.path.join(tdir, JSON_FILE)
with open(jsonfile, 'wt') as jfile:
json.dump(param_data, jfile)
sender = PDLSender(properties=props,
local_files=[jsonfile],
product_properties={'name': 'test'})
nfiles, msg = sender.send()
shutil.rmtree(tdir)
return (nfiles, msg)
def test_cancel():
props = {'java': '/usr/bin/java',
'jarfile': '/home/ProductClient/ProductClient.jar',
'privatekey': '/home/ProductClient/pdlkey',
'configfile': '/home/ProductClient/config.ini',
'source': 'ci',
'type': 'dummy',
'code': 'ci2015abcd',
'testint': 5,
'eventsource': 'us',
'eventsourcecode': 'us1234abcd'}
optional_props = {'latitude': 34.123,
'longitude': -188.456,
'depth': 10.1,
'eventtime': datetime.datetime.utcnow(),
'magnitude': 5.4}
product_props = {'maxmmi': 5.4,
'alert': 'yellow'}
props.update(optional_props)
thisfile = os.path.abspath(__file__)
patchfunc = 'impactutils.transfer.pdlsender.get_command_output'
with mock.patch(patchfunc) as mock_output:
pdl = PDLSender(properties=props, local_files=[thisfile],
product_properties=product_props)
# error code, stdout, stderr
mock_output.return_value = (True, b'stuff cancelled', b'')
stdout = pdl.cancel()
assert 'cancelled' in stdout.decode('utf-8')
def test_cancel_fail():
props = {'java': '',
'jarfile': '',
'privatekey': '',
'configfile': '',
'source': 'ci',
'type': 'dummy',
'code': 'ci2015abcd',
'eventsource': 'us',
'eventsourcecode': 'us1234abcd'}
optional_props = {'latitude': 34.123,
'longitude': -188.456,
'depth': 10.1,
'eventtime': datetime.datetime.utcnow(),
'magnitude': 5.4}
product_props = {'maxmmi': 5.4,
'alert': 'yellow'}
props.update(optional_props)
thisfile = os.path.abspath(__file__)
patchfunc = 'impactutils.transfer.pdlsender.get_command_output'
with mock.patch(patchfunc) as mock_output:
try:
# attempt to send two files
pdl = PDLSender(properties=props,
local_files=[thisfile],
product_properties=product_props)
# error code, stdout, stderr
mock_output.return_value = (False, b'error', b'')
_ = pdl.cancel()
except Exception as e:
assert 'Could not delete product' in str(e)
def _test_real_pdl(directory):
dirpath = pathlib.Path(directory)
props = {}
props['java'] = which('java')
props['jarfile'] = str(dirpath / 'ProductClient.jar')
props['privatekey'] = str(dirpath / 'pdlkey')
props['configfile'] = str(dirpath / 'config.ini')
props['source'] = 'nc'
props['type'] = 'dummy'
props['code'] = '73328466'
props['eventsource'] = 'nc'
props['eventsourcecode'] = 'nc73328466'
pprops = {}
pprops['string'] = 'green'
pprops['float'] = 5.1
pprops['int'] = 4
pprops['time'] = datetime.datetime.utcnow()
pdl = PDLSender(properties=props,
product_properties=pprops)
try:
nfiles, stdout = pdl.send()
print(f'Sent {nfiles} files with output: "{stdout}"')
except Exception as e:
print(str(e))
sys.exit(1)
try:
stdout = pdl.cancel()
print(f'Sent cancel message with output: "{stdout}"')
except Exception as e:
print(str(e))
sys.exit(1)
def delete_fault(configfile, eventsource, eventsourcecode, jarfile, java,
privatekey, product_source, number=None):
"""Delete finite fault product.
Args:
configfile (str): Location of PDL config file.
eventsource (str): Network that originated the event.
eventsourcecode (str): Event code.
jarfile (str): Location of PDL jar file.
java (str): Location of Java binary.
privatekey (str): Location of PDL private key.
product_source (str): Network contributing this product to ComCat.
number (int): Number of product (used for two plane solutions).
Default is None.
Returns:
str: Message with any error information.
"""
props = {}
props['java'] = java
props['jarfile'] = jarfile
props['privatekey'] = privatekey
props['configfile'] = configfile
props['source'] = product_source
props['eventsource'] = eventsource
props['eventsourcecode'] = eventsourcecode
props['code'] = eventsource + eventsourcecode
if number is not None:
props['code'] += '_' + str(number)
props['type'] = 'finite-fault'
sender = PDLSender(properties=props)
msg = sender.cancel()
return (msg)
def test_cancel():
props = {
"java": "/usr/bin/java",
"jarfile": "/home/ProductClient/ProductClient.jar",
"privatekey": "/home/ProductClient/pdlkey",
"configfile": "/home/ProductClient/config.ini",
"source": "ci",
"type": "dummy",
"code": "ci2015abcd",
"testint": 5,
"eventsource": "us",
"eventsourcecode": "us1234abcd",
}
optional_props = {
"latitude": 34.123,
"longitude": -188.456,
"depth": 10.1,
"eventtime": datetime.datetime.utcnow(),
"magnitude": 5.4,
}
product_props = {"maxmmi": 5.4, "alert": "yellow"}
props.update(optional_props)
thisfile = os.path.abspath(__file__)
patchfunc = "impactutils.transfer.pdlsender.get_command_output"
with mock.patch(patchfunc) as mock_output:
pdl = PDLSender(properties=props,
local_files=[thisfile],
product_properties=product_props)
# error code, stdout, stderr
mock_output.return_value = (True, b"stuff cancelled", b"")
stdout = pdl.cancel()
assert "cancelled" in stdout.decode("utf-8")
def _test_real_pdl(directory):
dirpath = pathlib.Path(directory)
props = {}
props["java"] = which("java")
props["jarfile"] = str(dirpath / "ProductClient.jar")
props["privatekey"] = str(dirpath / "pdlkey")
props["configfile"] = str(dirpath / "config.ini")
props["source"] = "nc"
props["type"] = "dummy"
props["code"] = "73328466"
props["eventsource"] = "nc"
props["eventsourcecode"] = "nc73328466"
pprops = {}
pprops["string"] = "green"
pprops["float"] = 5.1
pprops["int"] = 4
pprops["time"] = datetime.datetime.utcnow()
pdl = PDLSender(properties=props, product_properties=pprops)
try:
nfiles, stdout = pdl.send()
print(f'Sent {nfiles} files with output: "{stdout}"')
except Exception as e:
print(str(e))
sys.exit(1)
try:
stdout = pdl.cancel()
print(f'Sent cancel message with output: "{stdout}"')
except Exception as e:
print(str(e))
sys.exit(1)
def execute(self):
# call parent execute() method
# this will set the self.info and self.config
# dictionaries, and self.datadir
super(PDLTransfer, self).execute()
# check to see if PDL is a configured method
if 'pdl' not in self.config:
logging.info('No PDL transfer has been configured. Returning.')
return
# do PDL specific stuff
pdl_dir = os.path.join(self.datadir, 'pdl')
products_dir = os.path.join(self.datadir, 'products')
if not os.path.isdir(pdl_dir):
raise NotADirectoryError('%s does not exist.' % pdl_dir)
# get the properties needed for the sender
properties, product_properties = self.getProperties(self.info)
downloads_dir = os.path.join(pdl_dir, 'download')
if os.path.isdir(downloads_dir):
shutil.rmtree(downloads_dir, ignore_errors=True)
shutil.copytree(products_dir, downloads_dir)
# loop over all possible pdl destinations, send products to
# each one
for destination, params in self.config['pdl'].items():
params.update(properties)
if self.usedevconfig is True:
if (params['devconfig'] is None
or not os.path.isfile(params['devconfig'])):
raise FileNotFoundError('Dev config file "%s" does not '
'exist' % params['devconfig'])
# Swap the config file for the devconfig file
params['configfile'] = params['devconfig']
fmt = 'Doing PDL transfer to %s DEV...' % destination
logging.debug(fmt)
else:
fmt = 'Doing PDL transfer to %s...' % destination
logging.debug(fmt)
sender = PDLSender(properties=params,
local_directory=pdl_dir,
product_properties=product_properties)
if self.cancel:
msg = sender.cancel()
else:
nfiles, msg = sender.send()
fmt = '%i files sent. Message from sender: \n"%s"'
tpl = (nfiles, msg)
logging.info(fmt % tpl)
if not self.cancel:
shutil.rmtree(downloads_dir, ignore_errors=True)
def store_fault(configfile,
eventsource,
eventsourcecode,
jarfile,
java,
pdlfolder,
privatekey,
product_source,
properties,
number=None):
"""Store parametric data.
Args:
configfile (str): Location of PDL config file.
eventsource (str): Network that originated the event.
eventsourcecode (str): Event code.
jarfile (str): Location of PDL jar file.
java (str): Location of Java binary.
pdlfolder (str): Folder to send.
privatekey (str): Location of PDL private key.
product_source (str): Network contributing this product to ComCat.
properties (str): Dictionary of product properties.
number (int): Number of product (used for two plane solutions).
Default is None.
Returns:
int: Number of files transferred
str: Message with any error information.
"""
props = {}
props['java'] = java
props['jarfile'] = jarfile
props['privatekey'] = privatekey
props['configfile'] = configfile
props['source'] = product_source
props['eventsource'] = eventsource
props['eventsourcecode'] = eventsourcecode
props['code'] = eventsource + eventsourcecode
if number is not None:
props['code'] += '_' + str(number)
props['type'] = 'finite-fault'
sender = PDLSender(properties=props,
local_directory=pdlfolder,
product_properties=properties)
nfiles, msg = sender.send()
return (nfiles, msg)
def test_send():
props = {
"java": "/usr/bin/java",
"jarfile": "/home/ProductClient/ProductClient.jar",
"privatekey": "/home/ProductClient/pdlkey",
"configfile": "/home/ProductClient/config.ini",
"source": "ci",
"type": "dummy",
"code": "ci2015abcd",
"eventsource": "us",
"eventsourcecode": "us1234abcd",
}
optional_props = {
"latitude": 34.123,
"longitude": -188.456,
"depth": 10.1,
"eventtime": datetime.datetime.utcnow(),
"magnitude": 5.4,
}
product_props = {
"maxmmi": 5.4,
"time": datetime.datetime.utcnow(),
"other": "testme",
"testint": 5,
"alert": "yellow",
}
cmdline_args = {"signatureVersion": "v1"}
props.update(optional_props)
thisfile = os.path.abspath(__file__)
patchfunc = "impactutils.transfer.pdlsender.get_command_output"
with mock.patch(patchfunc) as mock_output:
pdl = PDLSender(
properties=props,
local_files=[thisfile],
product_properties=product_props,
cmdline_args=cmdline_args,
)
# error code, stdout, stderr
mock_output.return_value = (True, b"stuff sent", b"")
nfiles, send_msg = pdl.send()
assert nfiles == 1
assert '1 files sent successfully: resulting in output: "stuff sent"'
assert "stuff sent" in send_msg
def test_send():
props = {
'java': '',
'jarfile': '',
'privatekey': '',
'configfile': '',
'source': 'ci',
'type': 'dummy',
'code': 'ci2015abcd',
'eventsource': 'us',
'eventsourcecode': 'us1234abcd'
}
optional_props = {
'latitude': 34.123,
'longitude': -188.456,
'depth': 10.1,
'eventtime': datetime.datetime.utcnow(),
'magnitude': 5.4
}
product_props = {
'maxmmi': 5.4,
'time': datetime.datetime.utcnow(),
'other': {
'name': 'testme'
},
'testint': 5,
'alert': 'yellow'
}
props.update(optional_props)
thisfile = os.path.abspath(__file__)
patchfunc = 'impactutils.transfer.pdlsender.get_command_output'
with mock.patch(patchfunc) as mock_output:
pdl = PDLSender(properties=props,
local_files=[thisfile],
product_properties=product_props)
# error code, stdout, stderr
mock_output.return_value = (True, b'stuff sent', b'')
nfiles, send_msg = pdl.send()
assert 'stuff sent' in send_msg
def store_fault(configfile, eventsource, eventsourcecode, jarfile, java,
pdlfolder, privatekey, product_source, properties, reviewed,
number, dry_run=False, suppress=False):
"""Store finite fault product using pdl.
Args:
configfile (str): Location of PDL config file.
eventsource (str): Network that originated the event.
eventsourcecode (str): Event code.
jarfile (str): Location of PDL jar file.
java (str): Location of Java binary.
pdlfolder (str): Folder to send.
privatekey (str): Location of PDL private key.
product_source (str): Network contributing this product to ComCat.
properties (str): Dictionary of product properties.
number (int): Number of product (used to distinguish multiple solutions).
dry_run (bool): Should this be a dry run that only outputs the pdl command
without sending the product. Default is False.
suppress (bool): Suppress the number suffix on the event code.
Returns:
int: Number of files transferred
str: Message with any error information.
"""
props = {}
props['java'] = java
props['jarfile'] = jarfile
props['privatekey'] = privatekey
props['configfile'] = configfile
props['source'] = product_source
props['eventsource'] = eventsource
props['eventsourcecode'] = eventsourcecode
props['code'] = eventsource + eventsourcecode
if not suppress:
props['code'] += '_' + str(number)
props['type'] = 'finite-fault'
if reviewed:
properties["review-status"] = "reviewed"
sender = PDLSender(properties=props, local_directory=pdlfolder,
product_properties=properties)
if dry_run:
nfiles = 0
cmd = sender._pdlcmd
cmd = cmd.replace('[STATUS]', 'UPDATE')
cmd = sender._replace_required_properties(cmd)
cmd = sender._replace_files(cmd)
cmd = sender._replace_product_properties(cmd)
cmd = sender._replace_optional_properties(cmd)
msg = cmd
else:
nfiles, msg = sender.send()
return (nfiles, msg)
def test_cancel_fail():
props = {
"java": "",
"jarfile": "",
"privatekey": "",
"configfile": "",
"source": "ci",
"type": "dummy",
"code": "ci2015abcd",
"eventsource": "us",
"eventsourcecode": "us1234abcd",
}
optional_props = {
"latitude": 34.123,
"longitude": -188.456,
"depth": 10.1,
"eventtime": datetime.datetime.utcnow(),
"magnitude": 5.4,
}
product_props = {"maxmmi": 5.4, "alert": "yellow"}
props.update(optional_props)
thisfile = os.path.abspath(__file__)
patchfunc = "impactutils.transfer.pdlsender.get_command_output"
with mock.patch(patchfunc) as mock_output:
try:
# attempt to send two files
pdl = PDLSender(
properties=props,
local_files=[thisfile],
product_properties=product_props,
)
# error code, stdout, stderr
mock_output.return_value = (False, b"error", b"")
_ = pdl.cancel()
except Exception as e:
assert "Could not delete product" in str(e)
def _test_send(internalhub):
CONFIG = '''senders = sender1
logdirectory = [FOLDER]/log
loglevel = FINE
redirectconsole = false
enableTracker = false
[sender1]
type = gov.usgs.earthquake.distribution.SocketProductSender
host = [PDLHUB]
port = 11235'''
PDLURL = 'https://ehppdl1.cr.usgs.gov/ProductClient.zip'
javabin = spawn.find_executable('java')
tempdir = None
try:
tempdir = tempfile.mkdtemp()
fh = urllib.request.urlopen(PDLURL)
zipdata = fh.read()
fh.close()
zipf = io.BytesIO(zipdata)
myzip = zipfile.ZipFile(zipf, 'r')
jarfile = myzip.extract('ProductClient/ProductClient.jar', tempdir)
myzip.close()
zipf.close()
configtext = CONFIG.replace('[PDLHUB]', internalhub)
configtext = configtext.replace('[FOLDER]', tempdir)
configfile = os.path.join(tempdir, 'config.ini')
f = open(configfile, 'wt')
f.write(textwrap.dedent(configtext))
f.close()
key = RSA.generate(2048)
keyfile = os.path.join(tempdir, 'pdlkey')
f = open(keyfile, 'wb')
f.write(key.exportKey('PEM'))
f.close()
props = {
'java': javabin,
'jarfile': jarfile,
'privatekey': keyfile,
'configfile': configfile,
'source': 'ci',
'type': 'dummy',
'code': 'ci2015abcd',
'eventsource': 'us',
'eventsourcecode': 'us1234abcd'
}
optional_props = {
'latitude': 34.123,
'longitude': -188.456,
'depth': 10.1,
'eventtime': datetime.datetime.utcnow(),
'magnitude': 5.4
}
product_props = {'maxmmi': 5.4, 'alert': 'yellow'}
props.update(optional_props)
thisfile = os.path.abspath(__file__)
pdl = PDLSender(properties=props,
local_files=[thisfile],
product_properties=product_props)
print('Sending...')
nfiles, send_msg = pdl.send()
print(send_msg)
print('Deleting...')
delete_msg = pdl.delete()
print(send_msg)
except Exception as obj:
print(str(obj))
finally:
# remove temporary pdl folder with jarfile, config, and keyfile in it
if tempdir is not None:
shutil.rmtree(tempdir)