logging.Formatter("{} - %(levelname)s - %(message)s".format(sys.argv[0])))
# Sanity check input arguments ...
markings = [marking.split(":") for marking in arguments.marking]
markings = dict([(source, target) for source, target in markings])
couchdb = couchdb.Server()[arguments.couchdb_database]
# Load projects ...
for source in glob.glob(os.path.join(arguments.input_dir, "project-*.json")):
logging.info("Loading project %s", source)
project = json.load(open(source))
del project["_rev"]
if arguments.force and project["_id"] in couchdb:
del couchdb[project["_id"]]
couchdb.save(project)
# Load models ...
for source in glob.glob(os.path.join(arguments.input_dir, "model-*.json")):
logging.info("Loading model %s", source)
model = json.load(open(source))
del model["_rev"]
if model["marking"] in markings:
model["marking"] = markings[model["marking"]]
if arguments.force and model["_id"] in couchdb:
del couchdb[model["_id"]]
couchdb.save(model)
# Load array sets ...
for source in glob.glob(os.path.join(arguments.input_dir, "array-set-*.hdf5")):
logging.info("Loading array set %s", source)
logging.getLogger().handlers[0].setFormatter(logging.Formatter("{} - %(levelname)s - %(message)s".format(sys.argv[0])))
# Sanity check input arguments ...
markings = [marking.split(":") for marking in arguments.marking]
markings = dict([(source, target) for source, target in markings])
couchdb = couchdb.Server()[arguments.couchdb_database]
# Load projects ...
for source in glob.glob(os.path.join(arguments.input_dir, "project-*.json")):
logging.info("Loading project %s", source)
project = json.load(open(source))
del project["_rev"]
if arguments.force and project["_id"] in couchdb:
del couchdb[project["_id"]]
couchdb.save(project)
# Load models ...
for source in glob.glob(os.path.join(arguments.input_dir, "model-*.json")):
logging.info("Loading model %s", source)
model = json.load(open(source))
del model["_rev"]
if model["marking"] in markings:
model["marking"] = markings[model["marking"]]
if arguments.force and model["_id"] in couchdb:
del couchdb[model["_id"]]
couchdb.save(model)
# Load array sets ...
for source in glob.glob(os.path.join(arguments.input_dir, "array-set-*.hdf5")):
logging.info("Loading array set %s", source)
logging.Formatter("{} - %(levelname)s - %(message)s".format(sys.argv[0])))
# Sanity check input arguments ...
markings = [marking.split(":") for marking in arguments.marking]
markings = dict([(source, target) for source, target in markings])
couchdb = couchdb.Server()[arguments.couchdb_database]
# Load projects ...
for source in glob.glob(os.path.join(arguments.input_dir, "project-*.json")):
logging.info("Loading project %s", source)
project = json.load(open(source))
del project["_rev"]
if arguments.force and project["_id"] in couchdb:
del couchdb[project["_id"]]
couchdb.save(project)
# Load models ...
for source in glob.glob(os.path.join(arguments.input_dir, "model-*.json")):
logging.info("Loading model %s", source)
model = json.load(open(source))
del model["_rev"]
if model["marking"] in markings:
model["marking"] = markings[model["marking"]]
if arguments.force and model["_id"] in couchdb:
del couchdb[model["_id"]]
couchdb.save(model)
# Load array sets ...
for source in glob.glob(os.path.join(arguments.input_dir, "array-set-*.hdf5")):
logging.info("Loading array set %s", source)
logging.getLogger().handlers[0].setFormatter(logging.Formatter("{} - %(levelname)s - %(message)s".format(sys.argv[0])))
# Sanity check input arguments ...
markings = [marking.split(":") for marking in arguments.marking]
markings = dict([(source, target) for source, target in markings])
couchdb = couchdb.Server()[arguments.couchdb_database]
# Load projects ...
for source in glob.glob(os.path.join(arguments.input_dir, "project-*.json")):
logging.info("Loading project %s", source)
project = json.load(open(source))
del project["_rev"]
if arguments.force and project["_id"] in couchdb:
del couchdb[project["_id"]]
couchdb.save(project)
# Load models ...
for source in glob.glob(os.path.join(arguments.input_dir, "model-*.json")):
logging.info("Loading model %s", source)
model = json.load(open(source))
del model["_rev"]
if model["marking"] in markings:
model["marking"] = markings[model["marking"]]
if arguments.force and model["_id"] in couchdb:
del couchdb[model["_id"]]
couchdb.save(model)
# Load array sets ...
for source in glob.glob(os.path.join(arguments.input_dir, "array-set-*.hdf5")):
logging.info("Loading array set %s", source)
# get a view list of all pd ids
for row in couchdb.view("slycat/project_datas"):
logging.info('Testing PD: {0}'.format(str(row.id)))
pd_doc = couchdb.get(type="project_data",id=row.id)
delete_pd = True
# go through model list in pd and start cleaning
for model_id in pd_doc['mid']:
model_doc = couchdb.get(type="model",id=model_id)
if model_doc is not None:
if "model-type" in model_doc:
#log.error("testing model type:[%s]" % str(model_doc["model-type"]))
if model_doc["model-type"] == "parameter-image":
logging.info('Skipping deletion of parameter-image PD: {0}'.format(str(row.id)))
delete_pd = False
# clean up models that don't need pd
elif "project_data" in model_doc:
logging.info('Deleting non-parameter-image PD: {0}'.format(str(row.id)))
del model_doc["project_data"]
couchdb.save(model_doc)
# clean up models that don't have a type
elif "project_data" in model_doc:
logging.info('Deleting PD: {0} from model without a model-type'.format(str(row.id)))
del model_doc["project_data"]
couchdb.save(model_doc)
if delete_pd:
# delete project data that doesn't have an associated model
logging.info('Deleting PD with NO mid: {0}'.format(str(row.id)))
couchdb.delete(pd_doc)
logging.info('Done deleting project datas')
except Exception:
print(f'database <{db}> already exists')
# save map_reduce views
dir_path = os.path.dirname(os.path.realpath(__file__))
mapreduce_dir = os.path.join(dir_path,'design_cities.json')
mapreduce_doc = json.load(open(mapreduce_dir,'r'))
map_dbs = ['tweets_mixed']
for db in map_dbs:
couchdb = couchserver[db]
try:
if '_rev' in mapreduce_doc:
del mapreduce_doc['_rev']
couchdb.save(mapreduce_doc)
print(f'design/cities view is created in <Database {db}>')
except Exception:
_rev = couchdb['_design/cities']["_rev"]
mapreduce_doc["_rev"] = _rev
couchdb.save(mapreduce_doc)
print(f'design/cities view is updated in <Database {db}>')
# save analysis results
analysis_files = ['adelaide_analysis_result.json','brisbane_analysis_result.json',\
'melbourne_analysis_result.json','perth_analysis_result.json','sydney_analysis_result.json']
couchdb = couchserver['analysis_results']
for filename in analysis_files:
analysis_file = open(os.path.join(dir_path,filename),'r')
data = json.load(analysis_file)
analysis_file.close()