def load_from_databank(sourcejson, dataproviderjson, dry_run=False, overwrite=True, meta_only=False, file_dir = None):
print "Working on ", sourcejson['fields']['indicator']
dataorg = DataOrg.by_name(dataproviderjson['fields']['title'])
dataorgMeta = {
'description': dataproviderjson['fields']['description'],
'label': dataproviderjson['fields']['title']
}
if not dataorg:
dataorg = DataOrg(dataorgMeta)
db.session.add(dataorg)
#dataorg will update with id here
db.session.commit()
#get or create dataset
dataset = Dataset.by_label(sourcejson['fields']['indicator'])
description = "http://databank.edip-maps.net/admin/etldata/dataconnection/" + str(sourcejson['pk']) + "/"
modelDataset = {'dataset':
{
'label': sourcejson['fields']['indicator'],
'name': sourcejson['fields']['indicator'],
'description': description,
'dataType': sourcejson['fields']['data_type'],
'dataorg_id': dataorg.id
}
}
if not dataset:
#create one
dataset = Dataset(modelDataset['dataset'])
#dataset.ORoperations = dataproviderjson['fields'].get('ORoperations', {})
#dataset.data = dataproviderjson['fields'].get('mapping',{})
db.session.add(dataset)
else:
#dataset.ORoperations = dataproviderjson['fields'].get('ORoperations', {})
#dataset.data = dataproviderjson['fields'].get('mapping',{})
dataset.update(modelDataset['dataset'])
db.session.commit()
systemaccount = Account.by_id(1)
if dataset.source:
try:
print "trying to delete source"
print dataset.source
dataset.source.delete()
except Exception, e:
print "could not delete source", e
def load_from_databank(sourcejson,
dataproviderjson,
dry_run=False,
overwrite=True,
meta_only=False,
file_dir=None):
print "Working on ", sourcejson['fields']['indicator']
dataorg = DataOrg.by_name(dataproviderjson['fields']['title'])
dataorgMeta = {
'description': dataproviderjson['fields']['description'],
'label': dataproviderjson['fields']['title']
}
if not dataorg:
dataorg = DataOrg(dataorgMeta)
db.session.add(dataorg)
#dataorg will update with id here
db.session.commit()
#get or create dataset
dataset = Dataset.by_label(sourcejson['fields']['indicator'])
description = "http://databank.edip-maps.net/admin/etldata/dataconnection/" + str(
sourcejson['pk']) + "/"
modelDataset = {
'dataset': {
'label': sourcejson['fields']['indicator'],
'name': sourcejson['fields']['indicator'],
'description': description,
'dataType': sourcejson['fields']['data_type'],
'dataorg_id': dataorg.id
}
}
if not dataset:
#create one
dataset = Dataset(modelDataset['dataset'])
#dataset.ORoperations = dataproviderjson['fields'].get('ORoperations', {})
#dataset.data = dataproviderjson['fields'].get('mapping',{})
db.session.add(dataset)
else:
#dataset.ORoperations = dataproviderjson['fields'].get('ORoperations', {})
#dataset.data = dataproviderjson['fields'].get('mapping',{})
dataset.update(modelDataset['dataset'])
db.session.commit()
systemaccount = Account.by_id(1)
if dataset.source:
try:
print "trying to delete source"
print dataset.source
dataset.source.delete()
except Exception, e:
print "could not delete source", e
sys.exit()
djangodump = json.load(f)
f.close()
metadata = []
dataconnections = []
for obj in djangodump:
if obj['model'] == "etldata.metadata":
metadata.append(obj)
elif obj['model'] == "etldata.dataconnection":
dataconnections.append(obj)
for dataobj in dataconnections:
datasetflask = Dataset.by_label(dataobj['fields']['indicator'])
if not datasetflask:
print "Could not find ", dataobj['fields']['indicator']
continue
print "\nworking on ", datasetflask
for met in metadata:
if dataobj['fields']['metadata'] == met['pk']:
dataorgobj = DataOrg.by_name(met['fields']['title'])
if not dataorgobj:
#create new one
datasetflask = DataOrg({
"label":
met['fields']['title'],
"description":
met['fields']['description']
f.close()
metadata = []
dataconnections = []
for obj in djangodump:
if obj['model'] == "etldata.metadata":
metadata.append(obj)
elif obj['model'] == "etldata.dataconnection":
dataconnections.append(obj)
for dataobj in dataconnections:
datasetflask = Dataset.by_label(dataobj['fields']['indicator'])
if not datasetflask:
print "Could not find ", dataobj['fields']['indicator']
continue
print "\nworking on ", datasetflask
for met in metadata:
if dataobj['fields']['metadata'] == met['pk']:
dataorgobj = DataOrg.by_name(met['fields']['title'])
if not dataorgobj:
#create new one
datasetflask = DataOrg({"label":met['fields']['title'], "description":met['fields']['description']})
db.session.add(datasetflask)
datasetflask.dataorg = datasetflask
print "created a new one"
else:
