def handle(self, *args, **options): if len(args) != 2: raise CommandError( 'You must specify a file to dump and the pk of the type') with open(args[0], 'r') as f: output = f.read() issues = json.loads(output) type = Type.objects.get(pk=args[1]) for issue in issues: try: i = Issue( **{ 'lat': issue['lat'], 'lon': issue['lon'], 'description': issue['description'], 'type': type }) i.save() except (IntegrityError, pygeolib.GeocoderError): pass self.stdout.write('Successfully dump file to the database')
def on_data(self, data): print('Inside tweet issue') logger.info('Inside tweet issue') try: tweet = json.loads(data) title = tweet['text'] geo = tweet['geo'] coord = tweet['coordinates'] user = tweet['user']['screen_name'] sentiment = requests.post( 'http://text-processing.com/api/sentiment/', data = { 'text': title, } ).text if not sentiment: logger.info('Sentiment could not be determined [Text: ' + title +']') return True sentiJson = json.loads(sentiment) if sentiJson['label'] != 'neg': logger.info('Sentiment is not negative') print('Sentiment is not negative') return True logger.info('Sentiment is negative') print('Sentiment is negative') print('User handle: @' + user) grievance = Issue(title=title, content=title) grievance.twitter_handle = '@' + user grievance.status = IssueStatus.objects.filter(id=1).get() grievance.tweet = tweet try: grievance.save() logger.info('Saved') reply = '@%s Your issue has been registered with id %s' %(user, grievance.issue_id) s = api.update_status(reply, tweet['id']) except Exception as e: logger.exception('Saving to database failed: %s', e) traceback.print_exc() return True except Exception as e: logger.exception('Unknown exception occurred: %s', e)
def save_city(self, city): city_objs = self.retrieve_city(city) for object in city_objs: if 'id' in object and object['latitud'] and object['longitud']: issue = Issue() issue.lat = float(object['latitud']) issue.lon = float(object['longitud']) issue.address = object['direccion'] or ' ' issue.description = object['desperfectoTexto'].replace("Tipo de incidencia: ", '') or ' ' if object['desperfecto'] in self.crash: issue.type = models.Type.objects.get(pk=5) elif object['desperfecto'] in self.disturb: issue.type = models.Type.objects.get(pk=4) elif object['desperfecto'] in self.noise_polution: issue.type = models.Type.objects.get(pk=2) elif object['desperfecto'] in self.vandalism: issue.type = models.Type.objects.get(pk=1) else: issue.type = models.Type.objects.get(pk=9) issue.save()
def _load_issue(self, mets_file): _logger.debug("parsing issue mets file: %s" % mets_file) doc = etree.parse(mets_file) # get the mods for the issue div = doc.xpath('.//mets:div[@TYPE="np:issue"]', namespaces=ns)[0] dmdid = div.attrib['DMDID'] mods = dmd_mods(doc, dmdid) # set up a new Issue issue = Issue() issue.volume = mods.xpath( 'string(.//mods:detail[@type="volume"]/mods:number[1])', namespaces=ns).strip() issue.number = mods.xpath( 'string(.//mods:detail[@type="issue"]/mods:number[1])', namespaces=ns).strip() issue.edition = int( mods.xpath( 'string(.//mods:detail[@type="edition"]/mods:number[1])', namespaces=ns)) issue.edition_label = mods.xpath( 'string(.//mods:detail[@type="edition"]/mods:caption[1])', namespaces=ns).strip() # parse issue date date_issued = mods.xpath('string(.//mods:dateIssued)', namespaces=ns) issue.date_issued = datetime.strptime(date_issued, '%Y-%m-%d') # attach the Issue to the appropriate Title lccn = mods.xpath('string(.//mods:identifier[@type="lccn"])', namespaces=ns).strip() try: title = Title.objects.get(lccn=lccn) except Exception as e: url = settings.MARC_RETRIEVAL_URLFORMAT % lccn _logger.info("attempting to load marc record from %s", url) management.call_command('load_titles', url) title = Title.objects.get(lccn=lccn) issue.title = title issue.batch = self.current_batch issue.save() _logger.debug("saved issue: %s" % issue.url) notes = [] for mods_note in mods.xpath('.//mods:note', namespaces=ns): type = mods_note.xpath('string(./@type)') label = mods_note.xpath('string(./@displayLabel)') text = mods_note.xpath('string(.)') note = models.IssueNote(type=type, label=label, text=text) notes.append(note) issue.notes.set(notes, bulk=False) issue.save() # attach pages: lots of logging because it's expensive for page_div in div.xpath('.//mets:div[@TYPE="np:page"]', namespaces=ns): try: page = self._load_page(doc, page_div, issue) self.pages_processed += 1 except BatchLoaderException as e: _logger.exception(e) return issue