def main_script(): ch = logging.StreamHandler(stream=sys.stdout) ch.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) logger.root.addHandler(ch) logger.root.setLevel(logging.DEBUG) args = sys.argv if len(args) not in (4,): print "Usage: %s <entries file> <working directory> <script>" % (args[0]) print """Where entries file contains: [ {"class": "data_entry", "data": {"file1": {"class": "file_object", "file_name": null, "mime_type": null}}, "dataset": null, "id": null, "location_offset": null, "timestamp": "2013-03-20T09:13:41.567Z"} ]""" return(1) entries_file = args[1] cwd = args[2] script = args[3] # Validate parameters if not os.path.exists(entries_file): print "Config file not found: %s"%(entries_file) return(1) if not os.path.exists(cwd) and os.path.isdir(cwd): print "Working directory does not exist" return(1) # Create config object m = Marshaller() with open(entries_file) as f: results = json.load(f) results = m.dict_to_obj(results) print "Initial results" print "---------------" for result in results: print str(result) if script != None: with open(script) as f: script = f.read() results = run_script(script, cwd, results) print "Processed results" print "-----------------" for result in results: print str(result) return 0
class ManagementService(xmlrpc.XMLRPC): """ An example object to be published. """ def __init__(self, staging_dir, service): """Initialise the management service. :param service: Service Facade instance being exposed by this XMLRPC service """ xmlrpc.XMLRPC.__init__(self, allowNone=True) self.service = service self.transaction_counter = 0 if not os.path.exists(staging_dir): raise ValueError("The provided staging directory doesn't exist") self.transactions = {} self.staging_dir = staging_dir self._marshaller = Marshaller() def xmlrpc_insert(self, obj): """ Insert the passed object into the ingester platform """ try: return self._marshaller.obj_to_dict(self.service.persist(self._marshaller.dict_to_obj(obj))) except IngestPlatformError, e: raise translate_exception(e) except Exception, e: logger.exception("Error inserting") raise xmlrpc.Fault(InternalSystemError.__xmlrpc_error__, str(e))
class ManagementService(xmlrpc.XMLRPC): """ An example object to be published. """ def __init__(self, staging_dir, service): """Initialise the management service. :param service: Service Facade instance being exposed by this XMLRPC service """ xmlrpc.XMLRPC.__init__(self, allowNone=True) self.service = service self.transaction_counter = 0 if not os.path.exists(staging_dir): raise ValueError("The provided staging directory doesn't exist") self.transactions = {} self.staging_dir = staging_dir self._marshaller = Marshaller() def xmlrpc_insert(self, obj): """ Insert the passed object into the ingester platform """ try: return self._marshaller.obj_to_dict( self.service.persist(self._marshaller.dict_to_obj(obj))) except IngestPlatformError, e: raise translate_exception(e) except Exception, e: logger.exception("Error inserting") raise xmlrpc.Fault(InternalSystemError.__xmlrpc_error__, str(e))
def __init__(self, staging_dir, service): """Initialise the management service. :param service: Service Facade instance being exposed by this XMLRPC service """ xmlrpc.XMLRPC.__init__(self, allowNone=True) self.service = service self.transaction_counter = 0 if not os.path.exists(staging_dir): raise ValueError("The provided staging directory doesn't exist") self.transactions = {} self.staging_dir = staging_dir self._marshaller = Marshaller()
def __init__(self, service, staging_dir, data_source_factory): """Create an ingester engine, and register itself with the service facade. """ self.service = service self.service.register_observation_listener(self) self.staging_dir = staging_dir if not os.path.exists(self.staging_dir): os.makedirs(self.staging_dir) self._ingress_queue = Queue.Queue() self._archive_queue = Queue.Queue() self._data_source_factory = data_source_factory self.running = True self.domain_marshaller = Marshaller()
from jcudc24ingesterapi.schemas import ConcreteSchema from jcudc24ingesterapi.models.locations import LocationOffset from jcudc24ingesterapi.ingester_platform_api import get_properties, Marshaller import datetime from jcudc24ingesterapi.models.data_sources import DatasetDataSource from jcudc24ingesterapi.ingester_exceptions import PersistenceError, \ InvalidObjectError, StaleObjectError from sqlalchemy.types import TEXT import json from jcudc24ingesterapi import ValidationError logger = logging.getLogger(__name__) Base = declarative_base() domain_marshaller = Marshaller() class Region(Base): __tablename__ = "REGION" __xmlrpc_class__ = "region" id = Column(Integer, primary_key=True) version = Column(Integer, nullable=False, default=1) name = Column(String(255)) # parentRegions = orm.relationship("Region") region_points = orm.relationship("RegionPoint") class RegionPoint(Base): __tablename__ = "REGION_POINT" id = Column(Integer, primary_key=True)
def setUp(self): unittest.TestCase.setUp(self) self.marshaller = Marshaller()
class TestMarshaller(unittest.TestCase): """Test marshalling and object round tripping""" def setUp(self): unittest.TestCase.setUp(self) self.marshaller = Marshaller() def test_schema_attributes(self): schema = DataEntryMetadataSchema() schema.addAttr(Double("one")) schema.addAttr(String("two")) self.assertEquals("one", schema.attrs["one"].name) self.assertEquals("two", schema.attrs["two"].name) self.assertTrue(isinstance(schema.attrs["one"], Double)) self.assertTrue(isinstance(schema.attrs["two"], String)) schema_dict = self.marshaller.obj_to_dict(schema) schema_obj = self.marshaller.dict_to_obj(schema_dict) self.assertEquals("one", schema_obj.attrs["one"].name) self.assertEquals("two", schema_obj.attrs["two"].name) self.assertTrue(isinstance(schema_obj.attrs["one"], Double)) self.assertTrue(isinstance(schema_obj.attrs["two"], String)) def test_dataset_roundtrip(self): """Attempt to round trip a dataset object""" script_contents = """Some Script More""" dataset = Dataset(location=1, schema=2, data_source=PullDataSource("http://www.bom.gov.au/radar/IDR733.gif", "file", processing_script=script_contents), location_offset=LocationOffset(0, 1, 2)) dataset_dict = self.marshaller.obj_to_dict(dataset) dataset1 = self.marshaller.dict_to_obj(dataset_dict) self.assertIsNotNone(dataset1, "Dataset should not be none") self.assertEquals(dataset1.location, dataset.location, "Location ID does not match") self.assertEquals(dataset1.schema, dataset.schema, "schema does not match %d!=%d"%(dataset1.schema, dataset.schema)) self.assertEquals(dataset1.location_offset.x, 0) self.assertEquals(dataset1.location_offset.y, 1) self.assertEquals(dataset1.location_offset.z, 2) def test_data_entry(self): dt = datetime.datetime.utcfromtimestamp(1357788112) dt = dt.replace(tzinfo = jcudc24ingesterapi.UTC) data_entry = DataEntry(1, dt) data_entry["temp"] = 1.2 data_entry_dto = self.marshaller.obj_to_dict(data_entry) self.assertEquals("2013-01-10T03:21:52.000Z", data_entry_dto["timestamp"]) self.assertEquals(1, data_entry_dto["dataset"]) self.assertEquals(1.2, data_entry_dto["data"]["temp"]) data_entry_return = self.marshaller.dict_to_obj(data_entry_dto) self.assertEquals(data_entry.timestamp, data_entry_return.timestamp) self.assertEquals(data_entry.dataset, data_entry_return.dataset) self.assertEquals(data_entry.data["temp"], data_entry_return.data["temp"]) def test_file_object_roundtrip(self): """The file object should marshall everything but the file stream""" data_entry = DataEntry(1) data_entry["temp"] = FileObject(f_path=os.path.join( os.path.dirname(jcudc24ingesterapi.__file__), "tests/test_ingest.xml"), mime_type="text/xml") data_entry_dto = self.marshaller.obj_to_dict(data_entry) self.assertEqual("text/xml", data_entry_dto["data"]["temp"]["mime_type"]) data_entry_domain = self.marshaller.dict_to_obj(data_entry_dto) self.assertEqual("text/xml", data_entry_domain["temp"].mime_type) def test_unit_of_work_roundtrip(self): unit = UnitOfWork(None) loc = Location(10, 11) loc.name = "Loc 1" unit.insert(loc) unit_dict = self.marshaller.obj_to_dict(unit) self.assertEquals("unit_of_work", unit_dict["class"]) unit2 = self.marshaller.dict_to_obj(unit_dict) self.assertEquals(10.0, unit2._to_insert[0].latitude) self.assertEquals(11.0, unit2._to_insert[0].longitude) def test_special_attr(self): loc = Location(10, 11) loc.correlationid = -1 loc_dict = self.marshaller.obj_to_dict([loc], special_attrs=["correlationid"]) self.assertEquals(1, len(loc_dict)) self.assertEquals(-1, loc_dict[0]["correlationid"]) def test_unit_of_work_validation(self): unit = UnitOfWork(None) loc = Location(10, 11) self.assertRaises(InvalidObjectError, unit.insert, loc) loc.name = "test" unit.insert(loc) # Should work now. def test_marshaller_data_entry_schema(self): schema = {'attributes': [{'units': None, 'description': None, 'name': 'file', 'class': 'file'}], 'id': None, 'class': 'data_entry_schema'} schema = self.marshaller.dict_to_obj(schema)
def main_ingress(): ch = logging.StreamHandler(stream=sys.stdout) ch.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) logger.root.addHandler(ch) logger.root.setLevel(logging.DEBUG) args = sys.argv if len(args) not in (3,4): print "Usage: %s <config file> <working directory>"%(args[0]) print """Where config file contains: { "class":"INGESTER_CLASS", "state":{...}, "parameters":{...}, "config":{...} }""" return(1) cfg_file = args[1] cwd = args[2] script = args[3] if len(args) > 3 else None # Validate parameters if not os.path.exists(cfg_file): print "Config file not found: %s"%(cfg_file) return(1) if not os.path.exists(cwd) and os.path.isdir(cwd): print "Working directory does not exist" return(1) with open(sys.argv[1], "r") as f: cfg = json.load(f) if "class" not in cfg or "state" not in cfg or "parameters" not in cfg or "config" not in cfg: print "Config file not valid" return(1) # Create config object m = Marshaller() data_source_do = m.class_for(cfg["class"])() for k in cfg["config"]: setattr(data_source_do, k, cfg["config"][k]) data_source = create_data_source(data_source_do, cfg["state"], cfg["parameters"]) results = data_source.fetch(cwd) print "Initial results" print "---------------" for result in results: print str(result) if script != None: with open(script) as f: script = f.read() results = run_script(script, cwd, results) print "Processed results" print "-----------------" for result in results: print str(result) with open(cfg_file, "wb") as _cfg: json.dump(cfg, _cfg) return 0
class TestMarshaller(unittest.TestCase): """Test marshalling and object round tripping""" def setUp(self): unittest.TestCase.setUp(self) self.marshaller = Marshaller() def test_schema_attributes(self): schema = DataEntryMetadataSchema() schema.addAttr(Double("one")) schema.addAttr(String("two")) self.assertEquals("one", schema.attrs["one"].name) self.assertEquals("two", schema.attrs["two"].name) self.assertTrue(isinstance(schema.attrs["one"], Double)) self.assertTrue(isinstance(schema.attrs["two"], String)) schema_dict = self.marshaller.obj_to_dict(schema) schema_obj = self.marshaller.dict_to_obj(schema_dict) self.assertEquals("one", schema_obj.attrs["one"].name) self.assertEquals("two", schema_obj.attrs["two"].name) self.assertTrue(isinstance(schema_obj.attrs["one"], Double)) self.assertTrue(isinstance(schema_obj.attrs["two"], String)) def test_dataset_roundtrip(self): """Attempt to round trip a dataset object""" script_contents = """Some Script More""" dataset = Dataset(location=1, schema=2, data_source=PullDataSource( "http://www.bom.gov.au/radar/IDR733.gif", "file", processing_script=script_contents), location_offset=LocationOffset(0, 1, 2)) dataset_dict = self.marshaller.obj_to_dict(dataset) dataset1 = self.marshaller.dict_to_obj(dataset_dict) self.assertIsNotNone(dataset1, "Dataset should not be none") self.assertEquals(dataset1.location, dataset.location, "Location ID does not match") self.assertEquals( dataset1.schema, dataset.schema, "schema does not match %d!=%d" % (dataset1.schema, dataset.schema)) self.assertEquals(dataset1.location_offset.x, 0) self.assertEquals(dataset1.location_offset.y, 1) self.assertEquals(dataset1.location_offset.z, 2) def test_data_entry(self): dt = datetime.datetime.utcfromtimestamp(1357788112) dt = dt.replace(tzinfo=jcudc24ingesterapi.UTC) data_entry = DataEntry(1, dt) data_entry["temp"] = 1.2 data_entry_dto = self.marshaller.obj_to_dict(data_entry) self.assertEquals("2013-01-10T03:21:52.000Z", data_entry_dto["timestamp"]) self.assertEquals(1, data_entry_dto["dataset"]) self.assertEquals(1.2, data_entry_dto["data"]["temp"]) data_entry_return = self.marshaller.dict_to_obj(data_entry_dto) self.assertEquals(data_entry.timestamp, data_entry_return.timestamp) self.assertEquals(data_entry.dataset, data_entry_return.dataset) self.assertEquals(data_entry.data["temp"], data_entry_return.data["temp"]) def test_file_object_roundtrip(self): """The file object should marshall everything but the file stream""" data_entry = DataEntry(1) data_entry["temp"] = FileObject(f_path=os.path.join( os.path.dirname(jcudc24ingesterapi.__file__), "tests/test_ingest.xml"), mime_type="text/xml") data_entry_dto = self.marshaller.obj_to_dict(data_entry) self.assertEqual("text/xml", data_entry_dto["data"]["temp"]["mime_type"]) data_entry_domain = self.marshaller.dict_to_obj(data_entry_dto) self.assertEqual("text/xml", data_entry_domain["temp"].mime_type) def test_unit_of_work_roundtrip(self): unit = UnitOfWork(None) loc = Location(10, 11) loc.name = "Loc 1" unit.insert(loc) unit_dict = self.marshaller.obj_to_dict(unit) self.assertEquals("unit_of_work", unit_dict["class"]) unit2 = self.marshaller.dict_to_obj(unit_dict) self.assertEquals(10.0, unit2._to_insert[0].latitude) self.assertEquals(11.0, unit2._to_insert[0].longitude) def test_special_attr(self): loc = Location(10, 11) loc.correlationid = -1 loc_dict = self.marshaller.obj_to_dict([loc], special_attrs=["correlationid"]) self.assertEquals(1, len(loc_dict)) self.assertEquals(-1, loc_dict[0]["correlationid"]) def test_unit_of_work_validation(self): unit = UnitOfWork(None) loc = Location(10, 11) self.assertRaises(InvalidObjectError, unit.insert, loc) loc.name = "test" unit.insert(loc) # Should work now. def test_marshaller_data_entry_schema(self): schema = { 'attributes': [{ 'units': None, 'description': None, 'name': 'file', 'class': 'file' }], 'id': None, 'class': 'data_entry_schema' } schema = self.marshaller.dict_to_obj(schema)
def main_script(): ch = logging.StreamHandler(stream=sys.stdout) ch.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) logger.root.addHandler(ch) logger.root.setLevel(logging.DEBUG) args = sys.argv if len(args) not in (4, ): print "Usage: %s <entries file> <working directory> <script>" % ( args[0]) print """Where entries file contains: [ {"class": "data_entry", "data": {"file1": {"class": "file_object", "file_name": null, "mime_type": null}}, "dataset": null, "id": null, "location_offset": null, "timestamp": "2013-03-20T09:13:41.567Z"} ]""" return (1) entries_file = args[1] cwd = args[2] script = args[3] # Validate parameters if not os.path.exists(entries_file): print "Config file not found: %s" % (entries_file) return (1) if not os.path.exists(cwd) and os.path.isdir(cwd): print "Working directory does not exist" return (1) # Create config object m = Marshaller() with open(entries_file) as f: results = json.load(f) results = m.dict_to_obj(results) print "Initial results" print "---------------" for result in results: print str(result) if script != None: with open(script) as f: script = f.read() results = run_script(script, cwd, results) print "Processed results" print "-----------------" for result in results: print str(result) return 0
def main_ingress(): ch = logging.StreamHandler(stream=sys.stdout) ch.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) logger.root.addHandler(ch) logger.root.setLevel(logging.DEBUG) args = sys.argv if len(args) not in (3, 4): print "Usage: %s <config file> <working directory>" % (args[0]) print """Where config file contains: { "class":"INGESTER_CLASS", "state":{...}, "parameters":{...}, "config":{...} }""" return (1) cfg_file = args[1] cwd = args[2] script = args[3] if len(args) > 3 else None # Validate parameters if not os.path.exists(cfg_file): print "Config file not found: %s" % (cfg_file) return (1) if not os.path.exists(cwd) and os.path.isdir(cwd): print "Working directory does not exist" return (1) with open(sys.argv[1], "r") as f: cfg = json.load(f) if "class" not in cfg or "state" not in cfg or "parameters" not in cfg or "config" not in cfg: print "Config file not valid" return (1) # Create config object m = Marshaller() data_source_do = m.class_for(cfg["class"])() for k in cfg["config"]: setattr(data_source_do, k, cfg["config"][k]) data_source = create_data_source(data_source_do, cfg["state"], cfg["parameters"]) results = data_source.fetch(cwd) print "Initial results" print "---------------" for result in results: print str(result) if script != None: with open(script) as f: script = f.read() results = run_script(script, cwd, results) print "Processed results" print "-----------------" for result in results: print str(result) with open(cfg_file, "wb") as _cfg: json.dump(cfg, _cfg) return 0