def load_from_databank(sourcejson, dataproviderjson, dry_run=False, overwrite=True, meta_only=False, file_dir = None): print "Working on ", sourcejson['fields']['indicator'] dataorg = DataOrg.by_name(dataproviderjson['fields']['title']) dataorgMeta = { 'description': dataproviderjson['fields']['description'], 'label': dataproviderjson['fields']['title'] } if not dataorg: dataorg = DataOrg(dataorgMeta) db.session.add(dataorg) #dataorg will update with id here db.session.commit() #get or create dataset dataset = Dataset.by_label(sourcejson['fields']['indicator']) description = "http://databank.edip-maps.net/admin/etldata/dataconnection/" + str(sourcejson['pk']) + "/" modelDataset = {'dataset': { 'label': sourcejson['fields']['indicator'], 'name': sourcejson['fields']['indicator'], 'description': description, 'dataType': sourcejson['fields']['data_type'], 'dataorg_id': dataorg.id } } if not dataset: #create one dataset = Dataset(modelDataset['dataset']) #dataset.ORoperations = dataproviderjson['fields'].get('ORoperations', {}) #dataset.data = dataproviderjson['fields'].get('mapping',{}) db.session.add(dataset) else: #dataset.ORoperations = dataproviderjson['fields'].get('ORoperations', {}) #dataset.data = dataproviderjson['fields'].get('mapping',{}) dataset.update(modelDataset['dataset']) db.session.commit() systemaccount = Account.by_id(1) if dataset.source: try: print "trying to delete source" print dataset.source dataset.source.delete() except Exception, e: print "could not delete source", e
def load_from_databank(sourcejson, dataproviderjson, dry_run=False, overwrite=True, meta_only=False, file_dir=None): print "Working on ", sourcejson['fields']['indicator'] dataorg = DataOrg.by_name(dataproviderjson['fields']['title']) dataorgMeta = { 'description': dataproviderjson['fields']['description'], 'label': dataproviderjson['fields']['title'] } if not dataorg: dataorg = DataOrg(dataorgMeta) db.session.add(dataorg) #dataorg will update with id here db.session.commit() #get or create dataset dataset = Dataset.by_label(sourcejson['fields']['indicator']) description = "http://databank.edip-maps.net/admin/etldata/dataconnection/" + str( sourcejson['pk']) + "/" modelDataset = { 'dataset': { 'label': sourcejson['fields']['indicator'], 'name': sourcejson['fields']['indicator'], 'description': description, 'dataType': sourcejson['fields']['data_type'], 'dataorg_id': dataorg.id } } if not dataset: #create one dataset = Dataset(modelDataset['dataset']) #dataset.ORoperations = dataproviderjson['fields'].get('ORoperations', {}) #dataset.data = dataproviderjson['fields'].get('mapping',{}) db.session.add(dataset) else: #dataset.ORoperations = dataproviderjson['fields'].get('ORoperations', {}) #dataset.data = dataproviderjson['fields'].get('mapping',{}) dataset.update(modelDataset['dataset']) db.session.commit() systemaccount = Account.by_id(1) if dataset.source: try: print "trying to delete source" print dataset.source dataset.source.delete() except Exception, e: print "could not delete source", e
sys.exit() djangodump = json.load(f) f.close() metadata = [] dataconnections = [] for obj in djangodump: if obj['model'] == "etldata.metadata": metadata.append(obj) elif obj['model'] == "etldata.dataconnection": dataconnections.append(obj) for dataobj in dataconnections: datasetflask = Dataset.by_label(dataobj['fields']['indicator']) if not datasetflask: print "Could not find ", dataobj['fields']['indicator'] continue print "\nworking on ", datasetflask for met in metadata: if dataobj['fields']['metadata'] == met['pk']: dataorgobj = DataOrg.by_name(met['fields']['title']) if not dataorgobj: #create new one datasetflask = DataOrg({ "label": met['fields']['title'], "description": met['fields']['description']
f.close() metadata = [] dataconnections = [] for obj in djangodump: if obj['model'] == "etldata.metadata": metadata.append(obj) elif obj['model'] == "etldata.dataconnection": dataconnections.append(obj) for dataobj in dataconnections: datasetflask = Dataset.by_label(dataobj['fields']['indicator']) if not datasetflask: print "Could not find ", dataobj['fields']['indicator'] continue print "\nworking on ", datasetflask for met in metadata: if dataobj['fields']['metadata'] == met['pk']: dataorgobj = DataOrg.by_name(met['fields']['title']) if not dataorgobj: #create new one datasetflask = DataOrg({"label":met['fields']['title'], "description":met['fields']['description']}) db.session.add(datasetflask) datasetflask.dataorg = datasetflask print "created a new one" else: