def store_params(param_data, config, eventsource, eventsourcecode): """Store parametric data in ComCat. Args: config (dict): Dictionary containing fields: - java Location of Java binary. - jarfile Location of PDL jar file. - privatekey Location of PDL private key. - configfile Location of PDL config file. - product_source Network contributing this product to ComCat. eventsource (str): Network that originated the event. eventsourcecode (str): Event code from network that originated the event. Returns: int: Number of files transferred (should always be 1) str: Message with any error information. """ props = {} props.update(config) props['source'] = props['product_source'] del props['product_source'] props['eventsource'] = eventsource props['eventsourcecode'] = eventsourcecode props['code'] = eventsource + eventsourcecode props['type'] = PRODUCT_TYPE tdir = tempfile.mkdtemp() jsonfile = os.path.join(tdir, JSON_FILE) with open(jsonfile, 'wt') as jfile: json.dump(param_data, jfile) sender = PDLSender(properties=props, local_files=[jsonfile], product_properties={'name': 'test'}) nfiles, msg = sender.send() shutil.rmtree(tdir) return (nfiles, msg)
def test_cancel(): props = {'java': '/usr/bin/java', 'jarfile': '/home/ProductClient/ProductClient.jar', 'privatekey': '/home/ProductClient/pdlkey', 'configfile': '/home/ProductClient/config.ini', 'source': 'ci', 'type': 'dummy', 'code': 'ci2015abcd', 'testint': 5, 'eventsource': 'us', 'eventsourcecode': 'us1234abcd'} optional_props = {'latitude': 34.123, 'longitude': -188.456, 'depth': 10.1, 'eventtime': datetime.datetime.utcnow(), 'magnitude': 5.4} product_props = {'maxmmi': 5.4, 'alert': 'yellow'} props.update(optional_props) thisfile = os.path.abspath(__file__) patchfunc = 'impactutils.transfer.pdlsender.get_command_output' with mock.patch(patchfunc) as mock_output: pdl = PDLSender(properties=props, local_files=[thisfile], product_properties=product_props) # error code, stdout, stderr mock_output.return_value = (True, b'stuff cancelled', b'') stdout = pdl.cancel() assert 'cancelled' in stdout.decode('utf-8')
def test_cancel_fail(): props = {'java': '', 'jarfile': '', 'privatekey': '', 'configfile': '', 'source': 'ci', 'type': 'dummy', 'code': 'ci2015abcd', 'eventsource': 'us', 'eventsourcecode': 'us1234abcd'} optional_props = {'latitude': 34.123, 'longitude': -188.456, 'depth': 10.1, 'eventtime': datetime.datetime.utcnow(), 'magnitude': 5.4} product_props = {'maxmmi': 5.4, 'alert': 'yellow'} props.update(optional_props) thisfile = os.path.abspath(__file__) patchfunc = 'impactutils.transfer.pdlsender.get_command_output' with mock.patch(patchfunc) as mock_output: try: # attempt to send two files pdl = PDLSender(properties=props, local_files=[thisfile], product_properties=product_props) # error code, stdout, stderr mock_output.return_value = (False, b'error', b'') _ = pdl.cancel() except Exception as e: assert 'Could not delete product' in str(e)
def _test_real_pdl(directory): dirpath = pathlib.Path(directory) props = {} props['java'] = which('java') props['jarfile'] = str(dirpath / 'ProductClient.jar') props['privatekey'] = str(dirpath / 'pdlkey') props['configfile'] = str(dirpath / 'config.ini') props['source'] = 'nc' props['type'] = 'dummy' props['code'] = '73328466' props['eventsource'] = 'nc' props['eventsourcecode'] = 'nc73328466' pprops = {} pprops['string'] = 'green' pprops['float'] = 5.1 pprops['int'] = 4 pprops['time'] = datetime.datetime.utcnow() pdl = PDLSender(properties=props, product_properties=pprops) try: nfiles, stdout = pdl.send() print(f'Sent {nfiles} files with output: "{stdout}"') except Exception as e: print(str(e)) sys.exit(1) try: stdout = pdl.cancel() print(f'Sent cancel message with output: "{stdout}"') except Exception as e: print(str(e)) sys.exit(1)
def delete_fault(configfile, eventsource, eventsourcecode, jarfile, java, privatekey, product_source, number=None): """Delete finite fault product. Args: configfile (str): Location of PDL config file. eventsource (str): Network that originated the event. eventsourcecode (str): Event code. jarfile (str): Location of PDL jar file. java (str): Location of Java binary. privatekey (str): Location of PDL private key. product_source (str): Network contributing this product to ComCat. number (int): Number of product (used for two plane solutions). Default is None. Returns: str: Message with any error information. """ props = {} props['java'] = java props['jarfile'] = jarfile props['privatekey'] = privatekey props['configfile'] = configfile props['source'] = product_source props['eventsource'] = eventsource props['eventsourcecode'] = eventsourcecode props['code'] = eventsource + eventsourcecode if number is not None: props['code'] += '_' + str(number) props['type'] = 'finite-fault' sender = PDLSender(properties=props) msg = sender.cancel() return (msg)
def test_cancel(): props = { "java": "/usr/bin/java", "jarfile": "/home/ProductClient/ProductClient.jar", "privatekey": "/home/ProductClient/pdlkey", "configfile": "/home/ProductClient/config.ini", "source": "ci", "type": "dummy", "code": "ci2015abcd", "testint": 5, "eventsource": "us", "eventsourcecode": "us1234abcd", } optional_props = { "latitude": 34.123, "longitude": -188.456, "depth": 10.1, "eventtime": datetime.datetime.utcnow(), "magnitude": 5.4, } product_props = {"maxmmi": 5.4, "alert": "yellow"} props.update(optional_props) thisfile = os.path.abspath(__file__) patchfunc = "impactutils.transfer.pdlsender.get_command_output" with mock.patch(patchfunc) as mock_output: pdl = PDLSender(properties=props, local_files=[thisfile], product_properties=product_props) # error code, stdout, stderr mock_output.return_value = (True, b"stuff cancelled", b"") stdout = pdl.cancel() assert "cancelled" in stdout.decode("utf-8")
def _test_real_pdl(directory): dirpath = pathlib.Path(directory) props = {} props["java"] = which("java") props["jarfile"] = str(dirpath / "ProductClient.jar") props["privatekey"] = str(dirpath / "pdlkey") props["configfile"] = str(dirpath / "config.ini") props["source"] = "nc" props["type"] = "dummy" props["code"] = "73328466" props["eventsource"] = "nc" props["eventsourcecode"] = "nc73328466" pprops = {} pprops["string"] = "green" pprops["float"] = 5.1 pprops["int"] = 4 pprops["time"] = datetime.datetime.utcnow() pdl = PDLSender(properties=props, product_properties=pprops) try: nfiles, stdout = pdl.send() print(f'Sent {nfiles} files with output: "{stdout}"') except Exception as e: print(str(e)) sys.exit(1) try: stdout = pdl.cancel() print(f'Sent cancel message with output: "{stdout}"') except Exception as e: print(str(e)) sys.exit(1)
def execute(self): # call parent execute() method # this will set the self.info and self.config # dictionaries, and self.datadir super(PDLTransfer, self).execute() # check to see if PDL is a configured method if 'pdl' not in self.config: logging.info('No PDL transfer has been configured. Returning.') return # do PDL specific stuff pdl_dir = os.path.join(self.datadir, 'pdl') products_dir = os.path.join(self.datadir, 'products') if not os.path.isdir(pdl_dir): raise NotADirectoryError('%s does not exist.' % pdl_dir) # get the properties needed for the sender properties, product_properties = self.getProperties(self.info) downloads_dir = os.path.join(pdl_dir, 'download') if os.path.isdir(downloads_dir): shutil.rmtree(downloads_dir, ignore_errors=True) shutil.copytree(products_dir, downloads_dir) # loop over all possible pdl destinations, send products to # each one for destination, params in self.config['pdl'].items(): params.update(properties) if self.usedevconfig is True: if (params['devconfig'] is None or not os.path.isfile(params['devconfig'])): raise FileNotFoundError('Dev config file "%s" does not ' 'exist' % params['devconfig']) # Swap the config file for the devconfig file params['configfile'] = params['devconfig'] fmt = 'Doing PDL transfer to %s DEV...' % destination logging.debug(fmt) else: fmt = 'Doing PDL transfer to %s...' % destination logging.debug(fmt) sender = PDLSender(properties=params, local_directory=pdl_dir, product_properties=product_properties) if self.cancel: msg = sender.cancel() else: nfiles, msg = sender.send() fmt = '%i files sent. Message from sender: \n"%s"' tpl = (nfiles, msg) logging.info(fmt % tpl) if not self.cancel: shutil.rmtree(downloads_dir, ignore_errors=True)
def store_fault(configfile, eventsource, eventsourcecode, jarfile, java, pdlfolder, privatekey, product_source, properties, number=None): """Store parametric data. Args: configfile (str): Location of PDL config file. eventsource (str): Network that originated the event. eventsourcecode (str): Event code. jarfile (str): Location of PDL jar file. java (str): Location of Java binary. pdlfolder (str): Folder to send. privatekey (str): Location of PDL private key. product_source (str): Network contributing this product to ComCat. properties (str): Dictionary of product properties. number (int): Number of product (used for two plane solutions). Default is None. Returns: int: Number of files transferred str: Message with any error information. """ props = {} props['java'] = java props['jarfile'] = jarfile props['privatekey'] = privatekey props['configfile'] = configfile props['source'] = product_source props['eventsource'] = eventsource props['eventsourcecode'] = eventsourcecode props['code'] = eventsource + eventsourcecode if number is not None: props['code'] += '_' + str(number) props['type'] = 'finite-fault' sender = PDLSender(properties=props, local_directory=pdlfolder, product_properties=properties) nfiles, msg = sender.send() return (nfiles, msg)
def test_send(): props = { "java": "/usr/bin/java", "jarfile": "/home/ProductClient/ProductClient.jar", "privatekey": "/home/ProductClient/pdlkey", "configfile": "/home/ProductClient/config.ini", "source": "ci", "type": "dummy", "code": "ci2015abcd", "eventsource": "us", "eventsourcecode": "us1234abcd", } optional_props = { "latitude": 34.123, "longitude": -188.456, "depth": 10.1, "eventtime": datetime.datetime.utcnow(), "magnitude": 5.4, } product_props = { "maxmmi": 5.4, "time": datetime.datetime.utcnow(), "other": "testme", "testint": 5, "alert": "yellow", } cmdline_args = {"signatureVersion": "v1"} props.update(optional_props) thisfile = os.path.abspath(__file__) patchfunc = "impactutils.transfer.pdlsender.get_command_output" with mock.patch(patchfunc) as mock_output: pdl = PDLSender( properties=props, local_files=[thisfile], product_properties=product_props, cmdline_args=cmdline_args, ) # error code, stdout, stderr mock_output.return_value = (True, b"stuff sent", b"") nfiles, send_msg = pdl.send() assert nfiles == 1 assert '1 files sent successfully: resulting in output: "stuff sent"' assert "stuff sent" in send_msg
def test_send(): props = { 'java': '', 'jarfile': '', 'privatekey': '', 'configfile': '', 'source': 'ci', 'type': 'dummy', 'code': 'ci2015abcd', 'eventsource': 'us', 'eventsourcecode': 'us1234abcd' } optional_props = { 'latitude': 34.123, 'longitude': -188.456, 'depth': 10.1, 'eventtime': datetime.datetime.utcnow(), 'magnitude': 5.4 } product_props = { 'maxmmi': 5.4, 'time': datetime.datetime.utcnow(), 'other': { 'name': 'testme' }, 'testint': 5, 'alert': 'yellow' } props.update(optional_props) thisfile = os.path.abspath(__file__) patchfunc = 'impactutils.transfer.pdlsender.get_command_output' with mock.patch(patchfunc) as mock_output: pdl = PDLSender(properties=props, local_files=[thisfile], product_properties=product_props) # error code, stdout, stderr mock_output.return_value = (True, b'stuff sent', b'') nfiles, send_msg = pdl.send() assert 'stuff sent' in send_msg
def store_fault(configfile, eventsource, eventsourcecode, jarfile, java, pdlfolder, privatekey, product_source, properties, reviewed, number, dry_run=False, suppress=False): """Store finite fault product using pdl. Args: configfile (str): Location of PDL config file. eventsource (str): Network that originated the event. eventsourcecode (str): Event code. jarfile (str): Location of PDL jar file. java (str): Location of Java binary. pdlfolder (str): Folder to send. privatekey (str): Location of PDL private key. product_source (str): Network contributing this product to ComCat. properties (str): Dictionary of product properties. number (int): Number of product (used to distinguish multiple solutions). dry_run (bool): Should this be a dry run that only outputs the pdl command without sending the product. Default is False. suppress (bool): Suppress the number suffix on the event code. Returns: int: Number of files transferred str: Message with any error information. """ props = {} props['java'] = java props['jarfile'] = jarfile props['privatekey'] = privatekey props['configfile'] = configfile props['source'] = product_source props['eventsource'] = eventsource props['eventsourcecode'] = eventsourcecode props['code'] = eventsource + eventsourcecode if not suppress: props['code'] += '_' + str(number) props['type'] = 'finite-fault' if reviewed: properties["review-status"] = "reviewed" sender = PDLSender(properties=props, local_directory=pdlfolder, product_properties=properties) if dry_run: nfiles = 0 cmd = sender._pdlcmd cmd = cmd.replace('[STATUS]', 'UPDATE') cmd = sender._replace_required_properties(cmd) cmd = sender._replace_files(cmd) cmd = sender._replace_product_properties(cmd) cmd = sender._replace_optional_properties(cmd) msg = cmd else: nfiles, msg = sender.send() return (nfiles, msg)
def test_cancel_fail(): props = { "java": "", "jarfile": "", "privatekey": "", "configfile": "", "source": "ci", "type": "dummy", "code": "ci2015abcd", "eventsource": "us", "eventsourcecode": "us1234abcd", } optional_props = { "latitude": 34.123, "longitude": -188.456, "depth": 10.1, "eventtime": datetime.datetime.utcnow(), "magnitude": 5.4, } product_props = {"maxmmi": 5.4, "alert": "yellow"} props.update(optional_props) thisfile = os.path.abspath(__file__) patchfunc = "impactutils.transfer.pdlsender.get_command_output" with mock.patch(patchfunc) as mock_output: try: # attempt to send two files pdl = PDLSender( properties=props, local_files=[thisfile], product_properties=product_props, ) # error code, stdout, stderr mock_output.return_value = (False, b"error", b"") _ = pdl.cancel() except Exception as e: assert "Could not delete product" in str(e)
def _test_send(internalhub): CONFIG = '''senders = sender1 logdirectory = [FOLDER]/log loglevel = FINE redirectconsole = false enableTracker = false [sender1] type = gov.usgs.earthquake.distribution.SocketProductSender host = [PDLHUB] port = 11235''' PDLURL = 'https://ehppdl1.cr.usgs.gov/ProductClient.zip' javabin = spawn.find_executable('java') tempdir = None try: tempdir = tempfile.mkdtemp() fh = urllib.request.urlopen(PDLURL) zipdata = fh.read() fh.close() zipf = io.BytesIO(zipdata) myzip = zipfile.ZipFile(zipf, 'r') jarfile = myzip.extract('ProductClient/ProductClient.jar', tempdir) myzip.close() zipf.close() configtext = CONFIG.replace('[PDLHUB]', internalhub) configtext = configtext.replace('[FOLDER]', tempdir) configfile = os.path.join(tempdir, 'config.ini') f = open(configfile, 'wt') f.write(textwrap.dedent(configtext)) f.close() key = RSA.generate(2048) keyfile = os.path.join(tempdir, 'pdlkey') f = open(keyfile, 'wb') f.write(key.exportKey('PEM')) f.close() props = { 'java': javabin, 'jarfile': jarfile, 'privatekey': keyfile, 'configfile': configfile, 'source': 'ci', 'type': 'dummy', 'code': 'ci2015abcd', 'eventsource': 'us', 'eventsourcecode': 'us1234abcd' } optional_props = { 'latitude': 34.123, 'longitude': -188.456, 'depth': 10.1, 'eventtime': datetime.datetime.utcnow(), 'magnitude': 5.4 } product_props = {'maxmmi': 5.4, 'alert': 'yellow'} props.update(optional_props) thisfile = os.path.abspath(__file__) pdl = PDLSender(properties=props, local_files=[thisfile], product_properties=product_props) print('Sending...') nfiles, send_msg = pdl.send() print(send_msg) print('Deleting...') delete_msg = pdl.delete() print(send_msg) except Exception as obj: print(str(obj)) finally: # remove temporary pdl folder with jarfile, config, and keyfile in it if tempdir is not None: shutil.rmtree(tempdir)