def main_script():
ch = logging.StreamHandler(stream=sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.root.addHandler(ch)
logger.root.setLevel(logging.DEBUG)
args = sys.argv
if len(args) not in (4,):
print "Usage: %s <entries file> <working directory> <script>" % (args[0])
print """Where entries file contains:
[
{"class": "data_entry",
"data": {"file1": {"class": "file_object",
"file_name": null,
"mime_type": null}},
"dataset": null,
"id": null,
"location_offset": null,
"timestamp": "2013-03-20T09:13:41.567Z"}
]"""
return(1)
entries_file = args[1]
cwd = args[2]
script = args[3]
# Validate parameters
if not os.path.exists(entries_file):
print "Config file not found: %s"%(entries_file)
return(1)
if not os.path.exists(cwd) and os.path.isdir(cwd):
print "Working directory does not exist"
return(1)
# Create config object
m = Marshaller()
with open(entries_file) as f:
results = json.load(f)
results = m.dict_to_obj(results)
print "Initial results"
print "---------------"
for result in results:
print str(result)
if script != None:
with open(script) as f:
script = f.read()
results = run_script(script, cwd, results)
print "Processed results"
print "-----------------"
for result in results:
print str(result)
return 0
def main_script():
ch = logging.StreamHandler(stream=sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.root.addHandler(ch)
logger.root.setLevel(logging.DEBUG)
args = sys.argv
if len(args) not in (4, ):
print "Usage: %s <entries file> <working directory> <script>" % (
args[0])
print """Where entries file contains:
[
{"class": "data_entry",
"data": {"file1": {"class": "file_object",
"file_name": null,
"mime_type": null}},
"dataset": null,
"id": null,
"location_offset": null,
"timestamp": "2013-03-20T09:13:41.567Z"}
]"""
return (1)
entries_file = args[1]
cwd = args[2]
script = args[3]
# Validate parameters
if not os.path.exists(entries_file):
print "Config file not found: %s" % (entries_file)
return (1)
if not os.path.exists(cwd) and os.path.isdir(cwd):
print "Working directory does not exist"
return (1)
# Create config object
m = Marshaller()
with open(entries_file) as f:
results = json.load(f)
results = m.dict_to_obj(results)
print "Initial results"
print "---------------"
for result in results:
print str(result)
if script != None:
with open(script) as f:
script = f.read()
results = run_script(script, cwd, results)
print "Processed results"
print "-----------------"
for result in results:
print str(result)
return 0
def main_ingress():
ch = logging.StreamHandler(stream=sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.root.addHandler(ch)
logger.root.setLevel(logging.DEBUG)
args = sys.argv
if len(args) not in (3,4):
print "Usage: %s <config file> <working directory>"%(args[0])
print """Where config file contains:
{
"class":"INGESTER_CLASS",
"state":{...},
"parameters":{...},
"config":{...}
}"""
return(1)
cfg_file = args[1]
cwd = args[2]
script = args[3] if len(args) > 3 else None
# Validate parameters
if not os.path.exists(cfg_file):
print "Config file not found: %s"%(cfg_file)
return(1)
if not os.path.exists(cwd) and os.path.isdir(cwd):
print "Working directory does not exist"
return(1)
with open(sys.argv[1], "r") as f:
cfg = json.load(f)
if "class" not in cfg or "state" not in cfg or "parameters" not in cfg or "config" not in cfg:
print "Config file not valid"
return(1)
# Create config object
m = Marshaller()
data_source_do = m.class_for(cfg["class"])()
for k in cfg["config"]:
setattr(data_source_do, k, cfg["config"][k])
data_source = create_data_source(data_source_do, cfg["state"], cfg["parameters"])
results = data_source.fetch(cwd)
print "Initial results"
print "---------------"
for result in results:
print str(result)
if script != None:
with open(script) as f:
script = f.read()
results = run_script(script, cwd, results)
print "Processed results"
print "-----------------"
for result in results:
print str(result)
with open(cfg_file, "wb") as _cfg: json.dump(cfg, _cfg)
return 0
def main_ingress():
ch = logging.StreamHandler(stream=sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.root.addHandler(ch)
logger.root.setLevel(logging.DEBUG)
args = sys.argv
if len(args) not in (3, 4):
print "Usage: %s <config file> <working directory>" % (args[0])
print """Where config file contains:
{
"class":"INGESTER_CLASS",
"state":{...},
"parameters":{...},
"config":{...}
}"""
return (1)
cfg_file = args[1]
cwd = args[2]
script = args[3] if len(args) > 3 else None
# Validate parameters
if not os.path.exists(cfg_file):
print "Config file not found: %s" % (cfg_file)
return (1)
if not os.path.exists(cwd) and os.path.isdir(cwd):
print "Working directory does not exist"
return (1)
with open(sys.argv[1], "r") as f:
cfg = json.load(f)
if "class" not in cfg or "state" not in cfg or "parameters" not in cfg or "config" not in cfg:
print "Config file not valid"
return (1)
# Create config object
m = Marshaller()
data_source_do = m.class_for(cfg["class"])()
for k in cfg["config"]:
setattr(data_source_do, k, cfg["config"][k])
data_source = create_data_source(data_source_do, cfg["state"],
cfg["parameters"])
results = data_source.fetch(cwd)
print "Initial results"
print "---------------"
for result in results:
print str(result)
if script != None:
with open(script) as f:
script = f.read()
results = run_script(script, cwd, results)
print "Processed results"
print "-----------------"
for result in results:
print str(result)
with open(cfg_file, "wb") as _cfg:
json.dump(cfg, _cfg)
return 0
