def main_script():
ch = logging.StreamHandler(stream=sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.root.addHandler(ch)
logger.root.setLevel(logging.DEBUG)
args = sys.argv
if len(args) not in (4,):
print "Usage: %s <entries file> <working directory> <script>" % (args[0])
print """Where entries file contains:
[
{"class": "data_entry",
"data": {"file1": {"class": "file_object",
"file_name": null,
"mime_type": null}},
"dataset": null,
"id": null,
"location_offset": null,
"timestamp": "2013-03-20T09:13:41.567Z"}
]"""
return(1)
entries_file = args[1]
cwd = args[2]
script = args[3]
# Validate parameters
if not os.path.exists(entries_file):
print "Config file not found: %s"%(entries_file)
return(1)
if not os.path.exists(cwd) and os.path.isdir(cwd):
print "Working directory does not exist"
return(1)
# Create config object
m = Marshaller()
with open(entries_file) as f:
results = json.load(f)
results = m.dict_to_obj(results)
print "Initial results"
print "---------------"
for result in results:
print str(result)
if script != None:
with open(script) as f:
script = f.read()
results = run_script(script, cwd, results)
print "Processed results"
print "-----------------"
for result in results:
print str(result)
return 0
class ManagementService(xmlrpc.XMLRPC):
"""
An example object to be published.
"""
def __init__(self, staging_dir, service):
"""Initialise the management service.
:param service: Service Facade instance being exposed by this XMLRPC service
"""
xmlrpc.XMLRPC.__init__(self, allowNone=True)
self.service = service
self.transaction_counter = 0
if not os.path.exists(staging_dir):
raise ValueError("The provided staging directory doesn't exist")
self.transactions = {}
self.staging_dir = staging_dir
self._marshaller = Marshaller()
def xmlrpc_insert(self, obj):
""" Insert the passed object into the ingester platform
"""
try:
return self._marshaller.obj_to_dict(self.service.persist(self._marshaller.dict_to_obj(obj)))
except IngestPlatformError, e:
raise translate_exception(e)
except Exception, e:
logger.exception("Error inserting")
raise xmlrpc.Fault(InternalSystemError.__xmlrpc_error__, str(e))
class ManagementService(xmlrpc.XMLRPC):
"""
An example object to be published.
"""
def __init__(self, staging_dir, service):
"""Initialise the management service.
:param service: Service Facade instance being exposed by this XMLRPC service
"""
xmlrpc.XMLRPC.__init__(self, allowNone=True)
self.service = service
self.transaction_counter = 0
if not os.path.exists(staging_dir):
raise ValueError("The provided staging directory doesn't exist")
self.transactions = {}
self.staging_dir = staging_dir
self._marshaller = Marshaller()
def xmlrpc_insert(self, obj):
""" Insert the passed object into the ingester platform
"""
try:
return self._marshaller.obj_to_dict(
self.service.persist(self._marshaller.dict_to_obj(obj)))
except IngestPlatformError, e:
raise translate_exception(e)
except Exception, e:
logger.exception("Error inserting")
raise xmlrpc.Fault(InternalSystemError.__xmlrpc_error__, str(e))
def __init__(self, staging_dir, service):
"""Initialise the management service.
:param service: Service Facade instance being exposed by this XMLRPC service
"""
xmlrpc.XMLRPC.__init__(self, allowNone=True)
self.service = service
self.transaction_counter = 0
if not os.path.exists(staging_dir):
raise ValueError("The provided staging directory doesn't exist")
self.transactions = {}
self.staging_dir = staging_dir
self._marshaller = Marshaller()
def __init__(self, service, staging_dir, data_source_factory):
"""Create an ingester engine, and register itself with the service facade.
"""
self.service = service
self.service.register_observation_listener(self)
self.staging_dir = staging_dir
if not os.path.exists(self.staging_dir): os.makedirs(self.staging_dir)
self._ingress_queue = Queue.Queue()
self._archive_queue = Queue.Queue()
self._data_source_factory = data_source_factory
self.running = True
self.domain_marshaller = Marshaller()
def __init__(self, staging_dir, service):
"""Initialise the management service.
:param service: Service Facade instance being exposed by this XMLRPC service
"""
xmlrpc.XMLRPC.__init__(self, allowNone=True)
self.service = service
self.transaction_counter = 0
if not os.path.exists(staging_dir):
raise ValueError("The provided staging directory doesn't exist")
self.transactions = {}
self.staging_dir = staging_dir
self._marshaller = Marshaller()
from jcudc24ingesterapi.schemas import ConcreteSchema
from jcudc24ingesterapi.models.locations import LocationOffset
from jcudc24ingesterapi.ingester_platform_api import get_properties, Marshaller
import datetime
from jcudc24ingesterapi.models.data_sources import DatasetDataSource
from jcudc24ingesterapi.ingester_exceptions import PersistenceError, \
InvalidObjectError, StaleObjectError
from sqlalchemy.types import TEXT
import json
from jcudc24ingesterapi import ValidationError
logger = logging.getLogger(__name__)
Base = declarative_base()
domain_marshaller = Marshaller()
class Region(Base):
__tablename__ = "REGION"
__xmlrpc_class__ = "region"
id = Column(Integer, primary_key=True)
version = Column(Integer, nullable=False, default=1)
name = Column(String(255))
# parentRegions = orm.relationship("Region")
region_points = orm.relationship("RegionPoint")
class RegionPoint(Base):
__tablename__ = "REGION_POINT"
id = Column(Integer, primary_key=True)
def setUp(self):
unittest.TestCase.setUp(self)
self.marshaller = Marshaller()
class TestMarshaller(unittest.TestCase):
"""Test marshalling and object round tripping"""
def setUp(self):
unittest.TestCase.setUp(self)
self.marshaller = Marshaller()
def test_schema_attributes(self):
schema = DataEntryMetadataSchema()
schema.addAttr(Double("one"))
schema.addAttr(String("two"))
self.assertEquals("one", schema.attrs["one"].name)
self.assertEquals("two", schema.attrs["two"].name)
self.assertTrue(isinstance(schema.attrs["one"], Double))
self.assertTrue(isinstance(schema.attrs["two"], String))
schema_dict = self.marshaller.obj_to_dict(schema)
schema_obj = self.marshaller.dict_to_obj(schema_dict)
self.assertEquals("one", schema_obj.attrs["one"].name)
self.assertEquals("two", schema_obj.attrs["two"].name)
self.assertTrue(isinstance(schema_obj.attrs["one"], Double))
self.assertTrue(isinstance(schema_obj.attrs["two"], String))
def test_dataset_roundtrip(self):
"""Attempt to round trip a dataset object"""
script_contents = """Some Script
More"""
dataset = Dataset(location=1, schema=2, data_source=PullDataSource("http://www.bom.gov.au/radar/IDR733.gif", "file", processing_script=script_contents), location_offset=LocationOffset(0, 1, 2))
dataset_dict = self.marshaller.obj_to_dict(dataset)
dataset1 = self.marshaller.dict_to_obj(dataset_dict)
self.assertIsNotNone(dataset1, "Dataset should not be none")
self.assertEquals(dataset1.location, dataset.location, "Location ID does not match")
self.assertEquals(dataset1.schema, dataset.schema, "schema does not match %d!=%d"%(dataset1.schema, dataset.schema))
self.assertEquals(dataset1.location_offset.x, 0)
self.assertEquals(dataset1.location_offset.y, 1)
self.assertEquals(dataset1.location_offset.z, 2)
def test_data_entry(self):
dt = datetime.datetime.utcfromtimestamp(1357788112)
dt = dt.replace(tzinfo = jcudc24ingesterapi.UTC)
data_entry = DataEntry(1, dt)
data_entry["temp"] = 1.2
data_entry_dto = self.marshaller.obj_to_dict(data_entry)
self.assertEquals("2013-01-10T03:21:52.000Z", data_entry_dto["timestamp"])
self.assertEquals(1, data_entry_dto["dataset"])
self.assertEquals(1.2, data_entry_dto["data"]["temp"])
data_entry_return = self.marshaller.dict_to_obj(data_entry_dto)
self.assertEquals(data_entry.timestamp, data_entry_return.timestamp)
self.assertEquals(data_entry.dataset, data_entry_return.dataset)
self.assertEquals(data_entry.data["temp"], data_entry_return.data["temp"])
def test_file_object_roundtrip(self):
"""The file object should marshall everything but the file stream"""
data_entry = DataEntry(1)
data_entry["temp"] = FileObject(f_path=os.path.join(
os.path.dirname(jcudc24ingesterapi.__file__), "tests/test_ingest.xml"), mime_type="text/xml")
data_entry_dto = self.marshaller.obj_to_dict(data_entry)
self.assertEqual("text/xml", data_entry_dto["data"]["temp"]["mime_type"])
data_entry_domain = self.marshaller.dict_to_obj(data_entry_dto)
self.assertEqual("text/xml", data_entry_domain["temp"].mime_type)
def test_unit_of_work_roundtrip(self):
unit = UnitOfWork(None)
loc = Location(10, 11)
loc.name = "Loc 1"
unit.insert(loc)
unit_dict = self.marshaller.obj_to_dict(unit)
self.assertEquals("unit_of_work", unit_dict["class"])
unit2 = self.marshaller.dict_to_obj(unit_dict)
self.assertEquals(10.0, unit2._to_insert[0].latitude)
self.assertEquals(11.0, unit2._to_insert[0].longitude)
def test_special_attr(self):
loc = Location(10, 11)
loc.correlationid = -1
loc_dict = self.marshaller.obj_to_dict([loc], special_attrs=["correlationid"])
self.assertEquals(1, len(loc_dict))
self.assertEquals(-1, loc_dict[0]["correlationid"])
def test_unit_of_work_validation(self):
unit = UnitOfWork(None)
loc = Location(10, 11)
self.assertRaises(InvalidObjectError, unit.insert, loc)
loc.name = "test"
unit.insert(loc) # Should work now.
def test_marshaller_data_entry_schema(self):
schema = {'attributes': [{'units': None, 'description': None, 'name': 'file', 'class': 'file'}], 'id': None, 'class': 'data_entry_schema'}
schema = self.marshaller.dict_to_obj(schema)
def main_ingress():
ch = logging.StreamHandler(stream=sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.root.addHandler(ch)
logger.root.setLevel(logging.DEBUG)
args = sys.argv
if len(args) not in (3,4):
print "Usage: %s <config file> <working directory>"%(args[0])
print """Where config file contains:
{
"class":"INGESTER_CLASS",
"state":{...},
"parameters":{...},
"config":{...}
}"""
return(1)
cfg_file = args[1]
cwd = args[2]
script = args[3] if len(args) > 3 else None
# Validate parameters
if not os.path.exists(cfg_file):
print "Config file not found: %s"%(cfg_file)
return(1)
if not os.path.exists(cwd) and os.path.isdir(cwd):
print "Working directory does not exist"
return(1)
with open(sys.argv[1], "r") as f:
cfg = json.load(f)
if "class" not in cfg or "state" not in cfg or "parameters" not in cfg or "config" not in cfg:
print "Config file not valid"
return(1)
# Create config object
m = Marshaller()
data_source_do = m.class_for(cfg["class"])()
for k in cfg["config"]:
setattr(data_source_do, k, cfg["config"][k])
data_source = create_data_source(data_source_do, cfg["state"], cfg["parameters"])
results = data_source.fetch(cwd)
print "Initial results"
print "---------------"
for result in results:
print str(result)
if script != None:
with open(script) as f:
script = f.read()
results = run_script(script, cwd, results)
print "Processed results"
print "-----------------"
for result in results:
print str(result)
with open(cfg_file, "wb") as _cfg: json.dump(cfg, _cfg)
return 0
def setUp(self):
unittest.TestCase.setUp(self)
self.marshaller = Marshaller()
class TestMarshaller(unittest.TestCase):
"""Test marshalling and object round tripping"""
def setUp(self):
unittest.TestCase.setUp(self)
self.marshaller = Marshaller()
def test_schema_attributes(self):
schema = DataEntryMetadataSchema()
schema.addAttr(Double("one"))
schema.addAttr(String("two"))
self.assertEquals("one", schema.attrs["one"].name)
self.assertEquals("two", schema.attrs["two"].name)
self.assertTrue(isinstance(schema.attrs["one"], Double))
self.assertTrue(isinstance(schema.attrs["two"], String))
schema_dict = self.marshaller.obj_to_dict(schema)
schema_obj = self.marshaller.dict_to_obj(schema_dict)
self.assertEquals("one", schema_obj.attrs["one"].name)
self.assertEquals("two", schema_obj.attrs["two"].name)
self.assertTrue(isinstance(schema_obj.attrs["one"], Double))
self.assertTrue(isinstance(schema_obj.attrs["two"], String))
def test_dataset_roundtrip(self):
"""Attempt to round trip a dataset object"""
script_contents = """Some Script
More"""
dataset = Dataset(location=1,
schema=2,
data_source=PullDataSource(
"http://www.bom.gov.au/radar/IDR733.gif",
"file",
processing_script=script_contents),
location_offset=LocationOffset(0, 1, 2))
dataset_dict = self.marshaller.obj_to_dict(dataset)
dataset1 = self.marshaller.dict_to_obj(dataset_dict)
self.assertIsNotNone(dataset1, "Dataset should not be none")
self.assertEquals(dataset1.location, dataset.location,
"Location ID does not match")
self.assertEquals(
dataset1.schema, dataset.schema,
"schema does not match %d!=%d" % (dataset1.schema, dataset.schema))
self.assertEquals(dataset1.location_offset.x, 0)
self.assertEquals(dataset1.location_offset.y, 1)
self.assertEquals(dataset1.location_offset.z, 2)
def test_data_entry(self):
dt = datetime.datetime.utcfromtimestamp(1357788112)
dt = dt.replace(tzinfo=jcudc24ingesterapi.UTC)
data_entry = DataEntry(1, dt)
data_entry["temp"] = 1.2
data_entry_dto = self.marshaller.obj_to_dict(data_entry)
self.assertEquals("2013-01-10T03:21:52.000Z",
data_entry_dto["timestamp"])
self.assertEquals(1, data_entry_dto["dataset"])
self.assertEquals(1.2, data_entry_dto["data"]["temp"])
data_entry_return = self.marshaller.dict_to_obj(data_entry_dto)
self.assertEquals(data_entry.timestamp, data_entry_return.timestamp)
self.assertEquals(data_entry.dataset, data_entry_return.dataset)
self.assertEquals(data_entry.data["temp"],
data_entry_return.data["temp"])
def test_file_object_roundtrip(self):
"""The file object should marshall everything but the file stream"""
data_entry = DataEntry(1)
data_entry["temp"] = FileObject(f_path=os.path.join(
os.path.dirname(jcudc24ingesterapi.__file__),
"tests/test_ingest.xml"),
mime_type="text/xml")
data_entry_dto = self.marshaller.obj_to_dict(data_entry)
self.assertEqual("text/xml",
data_entry_dto["data"]["temp"]["mime_type"])
data_entry_domain = self.marshaller.dict_to_obj(data_entry_dto)
self.assertEqual("text/xml", data_entry_domain["temp"].mime_type)
def test_unit_of_work_roundtrip(self):
unit = UnitOfWork(None)
loc = Location(10, 11)
loc.name = "Loc 1"
unit.insert(loc)
unit_dict = self.marshaller.obj_to_dict(unit)
self.assertEquals("unit_of_work", unit_dict["class"])
unit2 = self.marshaller.dict_to_obj(unit_dict)
self.assertEquals(10.0, unit2._to_insert[0].latitude)
self.assertEquals(11.0, unit2._to_insert[0].longitude)
def test_special_attr(self):
loc = Location(10, 11)
loc.correlationid = -1
loc_dict = self.marshaller.obj_to_dict([loc],
special_attrs=["correlationid"])
self.assertEquals(1, len(loc_dict))
self.assertEquals(-1, loc_dict[0]["correlationid"])
def test_unit_of_work_validation(self):
unit = UnitOfWork(None)
loc = Location(10, 11)
self.assertRaises(InvalidObjectError, unit.insert, loc)
loc.name = "test"
unit.insert(loc) # Should work now.
def test_marshaller_data_entry_schema(self):
schema = {
'attributes': [{
'units': None,
'description': None,
'name': 'file',
'class': 'file'
}],
'id':
None,
'class':
'data_entry_schema'
}
schema = self.marshaller.dict_to_obj(schema)
def main_script():
ch = logging.StreamHandler(stream=sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.root.addHandler(ch)
logger.root.setLevel(logging.DEBUG)
args = sys.argv
if len(args) not in (4, ):
print "Usage: %s <entries file> <working directory> <script>" % (
args[0])
print """Where entries file contains:
[
{"class": "data_entry",
"data": {"file1": {"class": "file_object",
"file_name": null,
"mime_type": null}},
"dataset": null,
"id": null,
"location_offset": null,
"timestamp": "2013-03-20T09:13:41.567Z"}
]"""
return (1)
entries_file = args[1]
cwd = args[2]
script = args[3]
# Validate parameters
if not os.path.exists(entries_file):
print "Config file not found: %s" % (entries_file)
return (1)
if not os.path.exists(cwd) and os.path.isdir(cwd):
print "Working directory does not exist"
return (1)
# Create config object
m = Marshaller()
with open(entries_file) as f:
results = json.load(f)
results = m.dict_to_obj(results)
print "Initial results"
print "---------------"
for result in results:
print str(result)
if script != None:
with open(script) as f:
script = f.read()
results = run_script(script, cwd, results)
print "Processed results"
print "-----------------"
for result in results:
print str(result)
return 0
def main_ingress():
ch = logging.StreamHandler(stream=sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.root.addHandler(ch)
logger.root.setLevel(logging.DEBUG)
args = sys.argv
if len(args) not in (3, 4):
print "Usage: %s <config file> <working directory>" % (args[0])
print """Where config file contains:
{
"class":"INGESTER_CLASS",
"state":{...},
"parameters":{...},
"config":{...}
}"""
return (1)
cfg_file = args[1]
cwd = args[2]
script = args[3] if len(args) > 3 else None
# Validate parameters
if not os.path.exists(cfg_file):
print "Config file not found: %s" % (cfg_file)
return (1)
if not os.path.exists(cwd) and os.path.isdir(cwd):
print "Working directory does not exist"
return (1)
with open(sys.argv[1], "r") as f:
cfg = json.load(f)
if "class" not in cfg or "state" not in cfg or "parameters" not in cfg or "config" not in cfg:
print "Config file not valid"
return (1)
# Create config object
m = Marshaller()
data_source_do = m.class_for(cfg["class"])()
for k in cfg["config"]:
setattr(data_source_do, k, cfg["config"][k])
data_source = create_data_source(data_source_do, cfg["state"],
cfg["parameters"])
results = data_source.fetch(cwd)
print "Initial results"
print "---------------"
for result in results:
print str(result)
if script != None:
with open(script) as f:
script = f.read()
results = run_script(script, cwd, results)
print "Processed results"
print "-----------------"
for result in results:
print str(result)
with open(cfg_file, "wb") as _cfg:
json.dump(cfg, _cfg)
return 0
