def load_from_databank(sourcejson, dataproviderjson, dry_run=False, overwrite=True, meta_only=False, file_dir = None):
print "Working on ", sourcejson['fields']['indicator']
dataorg = DataOrg.by_name(dataproviderjson['fields']['title'])
dataorgMeta = {
'description': dataproviderjson['fields']['description'],
'label': dataproviderjson['fields']['title']
}
if not dataorg:
dataorg = DataOrg(dataorgMeta)
db.session.add(dataorg)
#dataorg will update with id here
db.session.commit()
#get or create dataset
dataset = Dataset.by_label(sourcejson['fields']['indicator'])
description = "http://databank.edip-maps.net/admin/etldata/dataconnection/" + str(sourcejson['pk']) + "/"
modelDataset = {'dataset':
{
'label': sourcejson['fields']['indicator'],
'name': sourcejson['fields']['indicator'],
'description': description,
'dataType': sourcejson['fields']['data_type'],
'dataorg_id': dataorg.id
}
}
if not dataset:
#create one
dataset = Dataset(modelDataset['dataset'])
#dataset.ORoperations = dataproviderjson['fields'].get('ORoperations', {})
#dataset.data = dataproviderjson['fields'].get('mapping',{})
db.session.add(dataset)
else:
#dataset.ORoperations = dataproviderjson['fields'].get('ORoperations', {})
#dataset.data = dataproviderjson['fields'].get('mapping',{})
dataset.update(modelDataset['dataset'])
db.session.commit()
systemaccount = Account.by_id(1)
if dataset.source:
try:
print "trying to delete source"
print dataset.source
dataset.source.delete()
except Exception, e:
print "could not delete source", e
def output_json(**args):
""" Output JSON data """
outputfile = args.get('outputfile', None)
if len(outputfile) != 1:
print "You need to specific one and only one output file"
return
outputfile = outputfile[0]
#need to load in this order for relations
#metadataorg
#dataorg
#source
#sourcefile
#wrap up files
#dataset
outputobj = []
for metadataorg in MetadataOrg.all().all():
outputobj.append(metadataorg.to_json_dump())
for dataorg in DataOrg.all().all():
outputobj.append(dataorg.to_json_dump())
for source in Source.all().all():
outputobj.append(source.to_json_dump())
for sourcefile in SourceFile.all().all():
outputobj.append(sourcefile.to_json_dump())
for dataset in Dataset.all().all():
outputobj.append(dataset.to_json_dump())
with open(outputfile, 'wb') as f:
json.dump(outputobj, f)
print "success"
print "written to ", outputfile
def output_json(**args):
""" Output JSON data """
outputfile = args.get('outputfile', None)
if len(outputfile) != 1:
print "You need to specific one and only one output file"
return
outputfile = outputfile[0]
#need to load in this order for relations
#metadataorg
#dataorg
#source
#sourcefile
#wrap up files
#dataset
outputobj = []
for metadataorg in MetadataOrg.all().all():
outputobj.append(metadataorg.to_json_dump())
for dataorg in DataOrg.all().all():
outputobj.append(dataorg.to_json_dump())
for source in Source.all().all():
outputobj.append(source.to_json_dump())
for sourcefile in SourceFile.all().all():
outputobj.append(sourcefile.to_json_dump())
for dataset in Dataset.all().all():
outputobj.append(dataset.to_json_dump())
with open(outputfile, 'wb') as f:
json.dump(outputobj, f)
print "success"
print "written to ", outputfile
def dataorgs():
#page = request.args.get('page')
q = DataOrg.get_all_admin().all()
return jsonify(q, headers= {'Cache-Control' : 'no-cache'})
def dataorgs():
#page = request.args.get('page')
q = DataOrg.get_all_admin().all()
return jsonify(q, headers={'Cache-Control': 'no-cache'})
def reference_data():
dataorgs = fromModel(DataOrg.get_all().all())
return jsonify({
'dataTypes': sorted(DATATYPES, key=lambda d: d['label']),
'dataorgs': sorted(dataorgs, key=lambda d: d['label'])
})
def load_from_databank(sourcejson,
dataproviderjson,
dry_run=False,
overwrite=True,
meta_only=False,
file_dir=None):
print "Working on ", sourcejson['fields']['indicator']
dataorg = DataOrg.by_name(dataproviderjson['fields']['title'])
dataorgMeta = {
'description': dataproviderjson['fields']['description'],
'label': dataproviderjson['fields']['title']
}
if not dataorg:
dataorg = DataOrg(dataorgMeta)
db.session.add(dataorg)
#dataorg will update with id here
db.session.commit()
#get or create dataset
dataset = Dataset.by_label(sourcejson['fields']['indicator'])
description = "http://databank.edip-maps.net/admin/etldata/dataconnection/" + str(
sourcejson['pk']) + "/"
modelDataset = {
'dataset': {
'label': sourcejson['fields']['indicator'],
'name': sourcejson['fields']['indicator'],
'description': description,
'dataType': sourcejson['fields']['data_type'],
'dataorg_id': dataorg.id
}
}
if not dataset:
#create one
dataset = Dataset(modelDataset['dataset'])
#dataset.ORoperations = dataproviderjson['fields'].get('ORoperations', {})
#dataset.data = dataproviderjson['fields'].get('mapping',{})
db.session.add(dataset)
else:
#dataset.ORoperations = dataproviderjson['fields'].get('ORoperations', {})
#dataset.data = dataproviderjson['fields'].get('mapping',{})
dataset.update(modelDataset['dataset'])
db.session.commit()
systemaccount = Account.by_id(1)
if dataset.source:
try:
print "trying to delete source"
print dataset.source
dataset.source.delete()
except Exception, e:
print "could not delete source", e
for obj in djangodump:
if obj['model'] == "etldata.metadata":
metadata.append(obj)
elif obj['model'] == "etldata.dataconnection":
dataconnections.append(obj)
for dataobj in dataconnections:
datasetflask = Dataset.by_label(dataobj['fields']['indicator'])
if not datasetflask:
print "Could not find ", dataobj['fields']['indicator']
continue
print "\nworking on ", datasetflask
for met in metadata:
if dataobj['fields']['metadata'] == met['pk']:
dataorgobj = DataOrg.by_name(met['fields']['title'])
if not dataorgobj:
#create new one
datasetflask = DataOrg({
"label":
met['fields']['title'],
"description":
met['fields']['description']
})
db.session.add(datasetflask)
datasetflask.dataorg = datasetflask
print "created a new one"
else:
datasetflask.dataorg = dataorgobj
print "adding to existing"
db.session.commit()
def reference_data():
dataorgs = fromModel(DataOrg.get_all().all())
return jsonify({
'dataTypes': sorted(DATATYPES, key=lambda d: d['label']),
'dataorgs': sorted(dataorgs, key=lambda d: d['label'])
})
metadata = []
dataconnections = []
for obj in djangodump:
if obj['model'] == "etldata.metadata":
metadata.append(obj)
elif obj['model'] == "etldata.dataconnection":
dataconnections.append(obj)
for dataobj in dataconnections:
datasetflask = Dataset.by_label(dataobj['fields']['indicator'])
if not datasetflask:
print "Could not find ", dataobj['fields']['indicator']
continue
print "\nworking on ", datasetflask
for met in metadata:
if dataobj['fields']['metadata'] == met['pk']:
dataorgobj = DataOrg.by_name(met['fields']['title'])
if not dataorgobj:
#create new one
datasetflask = DataOrg({"label":met['fields']['title'], "description":met['fields']['description']})
db.session.add(datasetflask)
datasetflask.dataorg = datasetflask
print "created a new one"
else:
datasetflask.dataorg = dataorgobj
print "adding to existing"
db.session.commit()