예제 #1
0
    def _format_url(self, template, source, last_updated, today=None):
        date_format = '%m-%d-%Y'
        lo = dateparser(last_updated).strftime(date_format)
        hi = datetime.now().strftime(date_format)

        if today:
            hi = dateparser(today).strftime(date_format)

        return template.format(source, lo, hi)
예제 #2
0
def datetime_parser(date_string):
    """
        Parse any time string.  Use a custom timezone matching if 
        the original matching does not pull one out.
    """
    try:
        date = dateparser(date_string)
        if date.tzinfo is None:
            date = dateparser(date_string, tzinfos=tzd)
        return date
    except:
        raise ValueError("Could not parse date string!")
예제 #3
0
파일: openstack.py 프로젝트: okoye/zabuza
 def __init__(self, *args, **kwargs):
   '''
   Expected keyword arguments:
   id: token id string
   expires: ISO-8601 string representation of expiry time
   issued_at: ISO-8601 string representation of time of issue
   tenant: tenant name this token is associated with
   '''
   #Fail if any expected parameter is not available.
   self._id = kwargs['id']
   self._expires = dateparser(kwargs['expires'])
   self._issued_at = dateparser(kwargs['issued_at'])
   self._tenant = kwargs['tenant']
예제 #4
0
파일: views.py 프로젝트: xgds/xgds_notes2
def doImportNotes(request, sourceFile, tz, resource):
    dictreader = csv.DictReader(sourceFile)
    for row in dictreader:
        row['author'] = request.user
        if row['content'] or row['tags']:
            if 'first_name' in row and 'last_name' in row:
                if row['first_name'] and row['last_name']:
                    try:
                        row['author'] = User.objects.get(first_name=row['first_name'], last_name=row['last_name'])
                        del row['first_name']
                        del row['last_name']
                    except:
                        pass
        if row['event_time']:
            event_time = dateparser(row['event_time'])
            if tz != pytz.utc:
                localized_time = tz.localize(event_time)
                event_time = TimeUtil.timeZoneToUtc(localized_time)
            row['event_time'] = event_time 
        
        try:
            # TODO implement tags when ready
            del row['tags']
        except:
            pass
        
        NOTE_MODEL = Note.get()
        note = NOTE_MODEL(**row)
        note.creation_time = datetime.now(pytz.utc)
        note.modification_time = datetime.now(pytz.utc)
        
        if resource:
            note.position = getClosestPosition(timestamp=note.event_time, resource=resource)
        note.save()
예제 #5
0
def getGpxTrackSet(docroot, ns):
    trackCollection = []
    for trk in docroot.findall("ns:trk", ns):
        trackName = trk.find("ns:name", ns).text
        trackSegPointList = trk.find("ns:trkseg", ns)
        trackSegment = {"name": trackName}
        trackSegPoints = []
        foundTimeInTrackData = True
        for point in trackSegPointList:
            if point.find("ns:time", ns) is not None:
                time = dateparser(point.find("ns:time", ns).text)
                time = time.replace(tzinfo=None)
            else:
                foundTimeInTrackData = False
                time = "<no time>"
            trackPoint = {"lat": float(point.attrib["lat"]),
                          "lon": float(point.attrib["lon"]),
                          "ele": float(point.find("ns:ele", ns).text),
                          "time": time}
            trackSegPoints.append(trackPoint)
        trackSegment["foundTimeInTrackData"] = foundTimeInTrackData
        trackSegment["trackPoints"] = trackSegPoints
        trackCollection.append(trackSegment)

    return trackCollection
예제 #6
0
파일: arin.py 프로젝트: hile/seine
 def __parse_date_entry(self, data):
     if data is None:
         return None
     try:
         return dateparser(data['$'])
     except (ValueError, KeyError):
         raise WhoisError('Error parsing date from %s' % data)
예제 #7
0
def find_latest_weather_alert():
  try:
    request = urllib2.urlopen("http://alerts.weather.gov/cap/us.php?x=1")
    dom = xml.dom.minidom.parse(request)
    latest_update = datetime.datetime(1970,1,1) ## for comparing against the latest weather entry

    ## this is all necessary because the government didn't feel like sorting the XML feed
    ## so the timestamps are all over the place. We have to find the latest entry ourselves
    weather_entries = dom.getElementsByTagName('entry')
    cur_entry = 0

    for entry in weather_entries:
      urgency = entry.getElementsByTagName('cap:urgency')[0].childNodes[0].data
      severity = entry.getElementsByTagName('cap:severity')[0].childNodes[0].data
      updated = entry.getElementsByTagName('updated')[0].childNodes[0].data  
      updated = dateparser(updated)
      updated = (updated - updated.utcoffset()).replace(tzinfo=None)
      cur_entry+=1
      
      ## only get the latest immediate and severe alerts, too much spam otherwise
      if latest_update < updated and urgency=="Immediate" and severity=="Severe":
        latest_update = updated
        latest_entry = cur_entry
        alert_url = entry.getElementsByTagName('id')[0].childNodes[0].data 
        
    return latest_update, latest_entry, alert_url
  except Exception as inst:
    print "find_latest_weather_alert: " + str(inst)
    pass    
예제 #8
0
    def process(self):
        report = self.receive_message()
        self.event_date = None

        raw_report = utils.base64_decode(report.get("raw"))

        for row in raw_report.splitlines():

            row = row.strip()

            if row.startswith('; Last-Modified:'):
                self.event_date = row.split('; Last-Modified: ')[1].strip()
                self.event_date = dateparser(self.event_date)

            if row == "" or row.startswith(';'):
                continue

            row_splitted = row.split(';')
            network = row_splitted[0].strip()

            event = Event(report)

            event.add('source.network', network)
            event.add('extra', {'blocklist': row_splitted[1].strip()})
            if self.event_date:
                event.add('time.source', self.event_date.isoformat())

            event.add('classification.type', 'spam')
            event.add('raw', row)

            self.send_message(event)
        self.acknowledge_message()
예제 #9
0
def simplify_message(message):
    'transforms a message instance into built-in types (json encodable)'

    assert not message.is_multipart()
    headers = NS((k.replace('-', '_').lower(), decode(v))
                  for k, v in message.items())

    # if the @ is not found or after the first space
    sender = headers.get('from', '')
    at = sender.find('@')
    if sender and (at == -1 or sender.find(' ') < at):
        sender = sender.replace(' at ', '@')
        # do not make it any easier to spam?
        # headers['from'] = sender

    if sender:
        headers['from_hash'] = md5(sender.encode('utf-8')).hexdigest()

    date = headers.get('date')
    if date:
        utc = dateparser(date).astimezone(UTC)
        headers['date_utc'] = int(time.mktime(utc.timetuple()))

    message_id = headers.get('message_id')
    if message_id:
        message_id_hash = sha1(message_id.encode('utf-8')).hexdigest()
        headers['message_id_hash'] = message_id_hash

    return NS(headers=headers, payload = message.get_payload())
예제 #10
0
def main():

    from dateutil.parser import parse as dateparser

    reporter_date = datetime.today() - timedelta(2)
    if len(sys.argv) >= 2:
        reporter_date = dateparser(sys.argv[1])

    dbConnPool = _get_user_db_pool()
    dbStstcPool = _get_ststc_db_pool()
    reConn = _getReConn()
    toDate = date.today() - timedelta(2)
    toDate = reporter_date

    try:
        cacheCatReporter(dbStstcPool, reConn, toDate)
    except UnicodeEncodeError:
        pass
    schedule_info(dbConnPool, reConn)
    # page_index(dbStstcPool, reConn)

    reConn.set('ibbd-ststc-date', (toDate + timedelta(1)).strftime('%Y/%m/%d'
                                                                   ))
    del reConn
    dbConnPool.disconnect()
    dbStstcPool.disconnect()
    del dbConnPool
예제 #11
0
파일: fields.py 프로젝트: victor3rc/lux
 def _clean(self, value, instance):
     if not isinstance(value, date):
         try:
             value = dateparser(value)
         except Exception:
             raise ValidationError(
                 self.validation_error.format(value))
     return self.todate(value)
예제 #12
0
 def get_object(self):
     page = Page(slug=self.kwargs["slug"])
     version = self.kwargs.get("version")
     date = self.kwargs.get("date")
     if version:
         return page.versions.as_of(version=int(version))
     if date:
         return page.versions.as_of(date=dateparser(date))
예제 #13
0
 def get_object(self):
     page = Page(slug=self.kwargs['slug'])
     version = self.kwargs.get('version')
     date = self.kwargs.get('date')
     if version:
         return page.versions.as_of(version=int(version))
     if date:
         return page.versions.as_of(date=dateparser(date))
예제 #14
0
파일: date.py 프로젝트: quantmind/lux
def to_timestamp(dt):
    if isinstance(dt, str):
        dt = dateparser(dt)
    if isinstance(dt, date):
        return time.mktime(dt.timetuple())
    elif isinstance(dt, NUMBER):
        return dt
    elif dt is not None:
        raise ValueError(dt)
예제 #15
0
    def get_redirect_url(self, slug, file, **kwargs):
        page_file = PageFile(slug=slug, name=file)
        version = self.kwargs.get('version')
        date = self.kwargs.get('date')

        if version:
            page_file = page_file.versions.as_of(version=int(version))
        if date:
            page_file = page_file.versions.as_of(date=dateparser(date))

        return page_file.file.url
예제 #16
0
def checktime(tweet,time1,time2):
    # Get the time this tweet was tweeted
    if "postedTime" in tweet:
        tweettime=tweet['postedTime']
        tweettime=dateparser(tweettime) # Get the date
        td=timedelta(hours=-5) # Normalize times to EST (-5 hours to GMT as reported from GNIP)
        tweettime+=td # Add the time delta to tweet time to get time of tweet in EST 

        # If tweet time is between or the same as time1 and time2 then it is good
        if time1.date() <= tweettime.date() and tweettime.date() <= time2.date():
            return True
    return False
예제 #17
0
    def __init__(self, fname_pattern=None, timestr=None, fnames_only=False):
        self.fnames_only = fnames_only
        if fname_pattern and os.path.exists(fname_pattern):
            if os.path.isfile(fname_pattern):
                self.get_df(fname_pattern)
            elif os.path.isdir(fname_pattern):
                pass

        self.timestr = timestr
        self.current_time = dateparser(timestr)
        self.fname = os.path.join(datapath,
                                  self.current_time.strftime("%Y%m%d%H"))
        self.increment = timedelta(hours=1)
예제 #18
0
파일: views.py 프로젝트: mivanov/editkit
 def get_object(self):
     page_slug = self.kwargs.get('slug')
     try:
         page = Page.objects.get(slug=page_slug)
         tags = page.pagetagset
         version = self.kwargs.get('version')
         date = self.kwargs.get('date')
         if version:
             return tags.versions.as_of(version=int(version))
         if date:
             return tags.versions.as_of(date=dateparser(date))
     except (Page.DoesNotExist, PageTagSet.DoesNotExist):
         return None
예제 #19
0
def get_timerange(timevar='time_ago', time_ago='1_days',
                  date1=None, date2=None):
    """returns 2 unix timestamps for a time range, based either
    on "time_ago" string or on a pair of date strings.

    Options
    --------
    timevar (string): one of 'time_ago' [default] or 'date_range'
    time_ago (string): time ago string (see Note 1)
    date1 (string):  string for initial date (see Note 2)
    date2 (string):  string for final date (see Note 2)

    Notes:
    ------
    1.  The time_ago string as the form '%f_%s'  with the
        string one of ('minutes, 'hours', or 'days'), like
             time_ago='1.5_days'   time_ago='15_minutes'

    2: 'date1' and 'date2' are strings of the form
       "%Y-%m-%d %H:%M:%S"
    """
    if (timevar.lower().startswith('date') and
        date1 is not None and
        date2 is not None):
        tmin = mktime(dateparser(date1).timetuple())
        tmax = mktime(dateparser(date2).timetuple())
        if tmin > tmax:
            tmin, tmax = tmax, tmin
    else:
        tmax   = time()
        tval, tunit = time_ago.split('_')
        opts = {}
        opts[tunit] = float(tval)
        dt_max = datetime.fromtimestamp(tmax)
        dt_min = dt_max - timedelta(**opts)
        tmin   = mktime(dt_min.timetuple())
    return tmin, tmax
예제 #20
0
파일: views.py 프로젝트: samrose/sapling
    def get_object(self):
        page = Page(slug=slugify(self.kwargs["slug"]))  # A dummy page object.
        latest_page = page.history.most_recent()
        # Need to set the pk on the dummy page for correct MapData lookup.
        page.pk = latest_page.id
        page.name = latest_page.name
        self.page = page

        mapdata = MapData(page=page)
        version = self.kwargs.get("version")
        date = self.kwargs.get("date")
        if version:
            return mapdata.history.as_of(version=int(version))
        if date:
            return mapdata.history.as_of(date=dateparser(date))
예제 #21
0
 def set_metadata(self, obj, item, tags=[]):
     schema = obj.Schema()
     schema['title'].set(obj, item['title'])
     if schema.has_key('text'):
         schema['text'].set(obj, item['bodyText'])
     schema['creators'].set(obj, [item['creator']])
     effective_date = dateparser().parse(item['effective_date'])
     schema['effectiveDate'].set(obj, effective_date)
     schema['creation_date'].set(obj, effective_date)
     schema['modification_date'].set(obj, effective_date)
     schema['subject'].set(obj, item['tags'] + tags)
     if item.has_key('remote_url') and schema.has_key('remoteUrl'):
         schema['remoteUrl'].set(obj, item['remote_url'])
     if item.has_key('author'):
         schema['contributors'].set(obj, [item['author']])
     obj.reindexObject()
예제 #22
0
    def get_context_data(self, **kwargs):
        context = super(CompareView, self).get_context_data(**kwargs)

        if self.kwargs.get('date1'):
            # Using datetimes to display diff.
            date1 = self.kwargs.get('date1')
            date2 = self.kwargs.get('date2')
            # Query parameter list used in history compare view.
            dates = self.request.GET.getlist('date')
            if not dates:
                dates = [v for v in (date1, date2) if v]
            dates = [dateparser(v) for v in dates]
            old = min(dates)
            new = max(dates)
            new_version = get_versions(self.object).as_of(date=new)
            prev_version = new_version.version_info.version_number() - 1
            if len(dates) == 1 and prev_version > 0:
                old_version = get_versions(self.object).as_of(
                    version=prev_version)
            elif prev_version <= 0:
                old_version = None
            else:
                old_version = get_versions(self.object).as_of(date=old)
        else:
            # Using version numbers to display diff.
            version1 = self.kwargs.get('version1')
            version2 = self.kwargs.get('version2')
            # Query parameter list used in history compare view.
            versions = self.request.GET.getlist('version')
            if not versions:
                versions = [v for v in (version1, version2) if v]
            if not versions:
                raise Http404("Versions not specified")
            versions = [int(v) for v in versions]
            old = min(versions)
            new = max(versions)
            if len(versions) == 1:
                old = max(new - 1, 1)
            if old > 0:
                old_version = get_versions(self.object).as_of(version=old)
            else:
                old_version = None
            new_version = get_versions(self.object).as_of(version=new)

        context.update({'old': old_version, 'new': new_version})
        return context
예제 #23
0
def getGpxWaypointList(docroot, ns):
    wptList = []
    for wpt in docroot.findall("ns:wpt", ns):
        if wpt.find("ns:time", ns) is not None:
            time = dateparser(wpt.find("ns:time", ns).text)
        else:
            time = None
        wptData = {"time": time,
                   "name": wpt.find("ns:name", ns).text,
                   "lat": float(wpt.attrib["lat"]),
                   "lon": float(wpt.attrib["lon"]),
                   "ele": float(wpt.find("ns:ele", ns).text),
                   "desc": wpt.find("ns:desc", ns).text,
                   "cmt": wpt.find("ns:cmt", ns).text,
                   "markerAndColor": [s.strip() for s in wpt.find("ns:sym", ns).
                       text.split(",")]}
        wptList.append(wptData)

    return wptList
예제 #24
0
    def get_object(self):
        region = self.get_region()
        # A dummy page object.
        page = Page(
            slug=slugify(self.kwargs['slug']),
            region=region
        )
        latest_page = page.versions.most_recent()
        # Need to set the pk on the dummy page for correct MapData lookup.
        page.pk = latest_page.id
        page.name = latest_page.name
        self.page = page

        mapdata = MapData(page=page, region=region)
        version = self.kwargs.get('version')
        date = self.kwargs.get('date')
        if version:
            return mapdata.versions.as_of(version=int(version))
        if date:
            return mapdata.versions.as_of(date=dateparser(date))
예제 #25
0
def get_weather_alert_data(alert_url):
  try:
    request = urllib2.urlopen(alert_url)
    dom = xml.dom.minidom.parse(request)
    msgType = dom.getElementsByTagName('msgType')[0].childNodes[0].data
    note = dom.getElementsByTagName('description')[0].childNodes[0].data
    note = note.replace("\n"," ")
    pattern = re.compile("\s+")
    note = pattern.sub(" ", note)
    ##turning off the text for now because its too much spam
    note = ""
    
    event = dom.getElementsByTagName('event')[0].childNodes[0].data
    urgency = dom.getElementsByTagName('urgency')[0].childNodes[0].data
    severity = dom.getElementsByTagName('severity')[0].childNodes[0].data
    certainty = dom.getElementsByTagName('certainty')[0].childNodes[0].data
    senderName = dom.getElementsByTagName('senderName')[0].childNodes[0].data
    
    ## Use the "effective" value because "sent" changes every time 
    ## the document is retrieved
    updated = dom.getElementsByTagName('effective')[0].childNodes[0].data
    updated = dateparser(updated)
    updated = (updated - updated.utcoffset()).replace(tzinfo=None)
    ago = (datetime.datetime.utcnow() - updated).seconds/60


    short_url = tools.shorten_url(alert_url)
    ## old text, too verbose
    ##alert_text = "[%s] %s: %s Urgency: %s Severity: %s Certainty: %s | %s (%s minutes ago)" % (senderName, msgType, event, urgency, severity, certainty, note[0:170], ago)
    ## new text is self limiting to the IRC limit of 428 characters
    
    alert_text_start = "[%s] %s: %s" % (senderName, msgType, event)
    alert_text_end = "(%s minutes ago) [ %s ]" % (ago, short_url)
    
    alert_text = "%s | %s %s" % (alert_text_start, note[:425-(len(alert_text_start+alert_text_end))], alert_text_end)
    return alert_text
    
  except Exception as inst:
    print "get_weather_alert_data: " + str(inst)
    pass    
예제 #26
0
파일: views.py 프로젝트: xgds/xgds_planner2
def planExport(request, uuid, name, time=None, outputDirectory=None, isAjax=False):
    """
    Normally plan export urls are built up by the planExporter.url
    but some exporters (pml) can take a time.
    """
    dbPlan = getDbPlan(uuid)

    formatCode = request.GET.get('format')
    if formatCode is not None:
        # user explicitly specified e.g. '?format=kml'
        exporterClass = choosePlanExporter.PLAN_EXPORTERS_BY_FORMAT.get(formatCode)
        if exporterClass is None:
            return HttpResponseBadRequest('invalid export format %s' % formatCode)
    else:
        # filename ends with e.g. '.kml'
        exporterClass = None
        for entry in choosePlanExporter.PLAN_EXPORTERS:
            if name.endswith(entry.extension):
                exporterClass = entry.exporterClass
        if exporterClass is None:
            return HttpResponseBadRequest('could not infer export format to use: "format" query parameter not set and extension not recognized for filename "%s"' % name)

    exporter = exporterClass()
    if time:
        try:
            thetime = dateparser(time)
            context = DotDict({'startTime': thetime.astimezone(pytz.utc)})
            exporter.initPlan(dbPlan, context)
        except:
            pass
        
    if outputDirectory:
        # output the exported file to a directory
        exporter.exportDbPlanToPath(dbPlan, os.path.join(outputDirectory, name))
        return True
    elif not isAjax:
        return exporter.getHttpResponse(dbPlan, attachmentName=name)
    else:
        return exporter.getHttpResponse(dbPlan, attachmentName=None)
예제 #27
0
    def process(self):
        report = self.receive_message()
        self.event_date = None

        if report is None or not report.contains("raw"):
            self.acknowledge_message()
            return

        raw_report = utils.base64_decode(report.value("raw"))

        for row in raw_report.split('\n'):

            row = row.strip()

            if row.startswith('; Last-Modified:'):
                self.event_date = row.split('; Last-Modified: ')[1].strip()
                self.event_date = dateparser(self.event_date)

            if row == "" or row.startswith(';'):
                continue

            row_splitted = row.split(';')
            network = row_splitted[0].strip()
            extra = json.dumps({'blocklist': row_splitted[1].strip()})

            event = Event(report)

            event.add('source.network', network, sanitize=True)
            event.add('extra', extra, sanitize=True)
            if self.event_date:
                event.add('time.source', self.event_date.isoformat(),
                          sanitize=True)

            event.add('classification.type', u'spam')
            event.add('raw', row, sanitize=True)

            self.send_message(event)
        self.acknowledge_message()
예제 #28
0
    def extractFromJson(self, overWriteDateModified=True, overWriteUuid=True):
        if overWriteUuid:
            if not self.uuid:
                self.uuid = makeUuid()
                self.jsonPlan.uuid = self.uuid
            self.jsonPlan.serverId = self.pk
        if overWriteDateModified:
            self.jsonPlan.dateModified = (datetime.datetime
                                          .now(pytz.utc)
                                          .replace(microsecond=0)
                                          .isoformat())
            self.jsonPlan.dateModified = self.jsonPlan.dateModified[:-6]+'Z'

        self.name = self.jsonPlan.name
        self.jsonPlan.url = self.get_absolute_url()
        self.jsonPlan.serverId = self.pk
        self.dateModified = dateparser(self.jsonPlan.dateModified).replace(tzinfo=pytz.utc)
        plannerUsers = User.objects.filter(username=self.jsonPlan.creator)
        if plannerUsers:
            self.creator = plannerUsers[0]
        else:
            self.creator = None

        # fill in stats
        try:
            exporter = statsPlanExporter.StatsPlanExporter()
#             print ' about to do stats'
            stats = exporter.exportDbPlan(self)
            for f in ('numStations', 'numSegments', 'numCommands', 'lengthMeters', 'estimatedDurationSeconds'):
                setattr(self, f, stats[f])
            self.stats.numCommandsByType = stats["numCommandsByType"]
            self.summary = statsPlanExporter.getSummary(stats)
        except:
            logging.warning('extractFromJson: could not extract stats from plan %s',
                            self.uuid)
            raise  # FIX
        return self
예제 #29
0
def handlegeocsv(geo,tweet):
        if(geo==None):
            return ""

        tweettime=""
        tweetuser=""
        bodytext=""
        if "postedTime" in tweet:
            tweettime=tweet['postedTime']
            tweettime=dateparser(tweettime) # Get the date
            td=timedelta(hours=-5) # Normalize times to EST (-5 hours to GMT as reported from GNIP)
            tweettime+=td # Add the time delta to tweet time to get time of tweet in EST 
            tweettime=tweettime.strftime("%Y%m%d")
        if "actor" in tweet and "displayName" in tweet['actor']:
            tweetuser=tweet['actor']['displayName']
        if "body" in tweet:
            bodytext=tweet['body']
            #bodytext = re.sub(wspattern,' ',bodytext)
	#endstr=","+tweettime+","+tweetuser+","+bodytext
	endstr=","+tweettime

        if(geo['type']=="Point"):
            lat=str(geo['coordinates'][1])
            lon=str(geo['coordinates'][0])
            return lat.decode('utf-8')+","+lon.decode('utf-8')+endstr+"\n"
        elif(geo['type']=="Polygon"):
            coordinates=geo['coordinates']
            # Sanity check polygon is simple (only a bounding box)
            if(len(coordinates)!=1 or len(coordinates[0])!=4):
                error("polygon size is not correct")
            coordinates=coordinates[0] # Enter the simple polygon
            lllist=zip(coordinates[0],coordinates[1],coordinates[2],coordinates[3])
            lonavg=sum(lllist[0])/float(len(lllist[0]))
            latavg=sum(lllist[1])/float(len(lllist[1]))
            return str(round(lonavg,5)).decode('utf-8')+","+str(round(latavg,5)).decode('utf-8')+endstr+"\n"
        else:
            return ""
예제 #30
0
파일: parser_drop.py 프로젝트: nizq/intelmq
    def process(self):
        report = self.receive_message()
        self.event_date = None

        if report is None or not report.contains("raw"):
            self.acknowledge_message()
            return

        raw_report = utils.base64_decode(report.value("raw"))

        for row in raw_report.split("\n"):

            row = row.strip()

            if row.startswith("; Last-Modified:"):
                self.event_date = row.split("; Last-Modified: ")[1].strip()
                self.event_date = dateparser(self.event_date)

            if row == "" or row.startswith(";"):
                continue

            row_splitted = row.split(";")
            network = row_splitted[0].strip()

            event = Event(report)

            event.add("source.network", network)
            event.add("extra", {"blocklist": row_splitted[1].strip()})
            if self.event_date:
                event.add("time.source", self.event_date.isoformat())

            event.add("classification.type", "spam")
            event.add("raw", row)

            self.send_message(event)
        self.acknowledge_message()
예제 #31
0
def schedulePlans(request, redirect=True):
    flight = None
    lastPlanExecution = None
    pe = None
    if request.method == 'POST':
        try:
            pids = request.POST['planIds']
            planIds = []
            for item in pids.split(","):
                planIds.append(int(item))
            plans = PLAN_MODEL.get().objects.filter(id__in=planIds)

            if 'planExecutionId' in request.POST and request.POST['planExecutionId'] != '':
                pe = PLAN_EXECUTION_MODEL.get().objects.get(pk=int(request.POST['planExecutionId']))

            schedule_date_string = request.POST['schedule_date']
            original_schedule_date = None
            prefix = None
            if schedule_date_string:
                # read the date; it comes in as UTC
                original_schedule_date = dateparser(schedule_date_string)
                original_schedule_date = pytz.utc.localize(original_schedule_date)
                schedule_date = original_schedule_date
                if pe:
                    firstPlan = pe.plan
                else:
                    firstPlan = plans[0]
                local_date = utcToTimeZone(original_schedule_date, str(firstPlan.jsonPlan.site.alternateCrs.properties.timezone))
                prefix = "%04d%02d%02d" % (local_date.year, local_date.month, local_date.day)

            flight_name = request.POST['flight']
            
            # see if flight name matches prefix, if not go by the date
            if prefix and flight_name:
                if not flight_name.startswith(prefix):
                    flight_name = None
                    
            try:
                flight = FLIGHT_MODEL.get().objects.get(name=flight_name)
            except FLIGHT_MODEL.get().DoesNotExist:
                # see if we can look it up by date
                if original_schedule_date:
                    flights = FLIGHT_MODEL.get().objects.filter(name__startswith=prefix)
                    if flights:
                        # pick the first one
                        flight = flights[0]
                    else:
                        # it does not exist we better make one
                        prefix = prefix + "A"
                        groupFlight, created = GROUP_FLIGHT_MODEL.get().objects.get_or_create(name=prefix)
                        for vehicle in VEHICLE_MODEL.get().objects.all():
                            newFlight = FLIGHT_MODEL.get()()
                            newFlight.group = groupFlight
                            newFlight.vehicle = vehicle
                            newFlight.name = prefix + "_" + vehicle.name
                            newFlight.locked = False
                            newFlight.uuid = uuid4()
                            newFlight.save(force_insert=True)
                            if not flight:
                                flight = newFlight

            if flight:
                for plan in plans:
                    if not pe:
                        pe = PLAN_EXECUTION_MODEL.get()()
                    pe.planned_start_time = schedule_date
                    pe.flight = flight
                    pe.plan = plan
                    
                    if settings.XGDS_PLANNER2_SCHEDULE_EXTRAS_METHOD:
                        pe = getClassByName(settings.XGDS_PLANNER2_SCHEDULE_EXTRAS_METHOD)(request, pe)
                        
                    pe.save()
                    lastPlanExecution = pe
        except:
            traceback.print_exc()
            return HttpResponse(json.dumps({'Success':"False", 'msg': 'Plan not scheduled'}), content_type='application/json', status=406)
            pass
    if redirect:
        return HttpResponseRedirect(reverse('planner2_index'))
    else:
        if lastPlanExecution:
            return HttpResponse(json.dumps(lastPlanExecution.toSimpleDict(), cls=DatetimeJsonEncoder), content_type='application/json')
        return HttpResponse(json.dumps({'Success':"True", 'msg': 'Plan scheduled'}), content_type='application/json')
예제 #32
0
def wdmtoswmm5rdii(wdmpath, *dsns, **kwds):
    """Print out DSN data to the screen in SWMM5 RDII format.

    Parameters
    ----------
    wdmpath
        Path and WDM
        filename.
    dsns
        The Data Set Numbers in the WDM
        file.
    {start_date}
    {end_date}

    """
    start_date = kwds.setdefault("start_date", None)
    end_date = kwds.setdefault("end_date", None)

    # Need to make sure that all DSNs are the same interval and all are
    # within start and end dates.
    collect_tcodes = {}
    collect_tsteps = {}
    collect_keys = []
    for dsn in dsns:
        dsn_desc = _describedsn(wdmpath, dsn)
        collect_tcodes[dsn_desc["tcode"]] = 1
        collect_tsteps[dsn_desc["tstep"]] = 1
        if start_date:
            assert dateparser(start_date) >= dateparser(dsn_desc["start_date"])
        if end_date:
            assert dateparser(end_date) <= dateparser(dsn_desc["end_date"])
        collect_keys.append((dsn_desc["dsn"], dsn_desc["location"]))
    assert len(collect_tcodes) == 1
    assert len(collect_tsteps) == 1

    collect_tcodes = list(collect_tcodes.keys())[0]
    collect_tsteps = list(collect_tsteps.keys())[0]

    collected_start_dates = []
    collected_end_dates = []
    collected_ts = {}
    for dsn, location in collect_keys:
        tmp = WDM.read_dsn(wdmpath,
                           int(dsn),
                           start_date=start_date,
                           end_date=end_date)
        collected_start_dates.append(tmp.index[0])
        collected_end_dates.append(tmp.index[-1])
        collected_ts[(dsn, location)] = tmp.values

    maptcode = {1: 1, 2: 60, 3: 3600, 4: 86400}

    print("SWMM5")
    print("RDII dump of DSNS {0} from {1}".format(dsns, wdmpath))
    print(maptcode[collect_tcodes] * collect_tsteps)
    print(1)
    print("FLOW CFS")
    print(len(dsns))
    for dsn, location in collect_keys:
        print("{0}_{1}".format(dsn, location))
    print("Node Year Mon Day Hr Min Sec Flow")
    # Can pick any time series because they should all have the same interval
    # and start and end dates.
    for dex, date in enumerate(tmp.index):
        for dsn, location in collect_keys:
            print("{0}_{1} {2} {3:02} {4:02} {5:02} {6:02} {7:02} {8}".format(
                dsn,
                location,
                date.year,
                date.month,
                date.day,
                date.hour,
                date.minute,
                date.second,
                collected_ts[(dsn, location)][dex],
            ))
예제 #33
0
 def __init__(self, d):
     self.__dict__ = d
     self.resources = Resource(d.get("resources"))
     self.mentions = [Mention(i) for i in d.get("mentions", [])] or None
     self.created_at = dateparser(d.get("created_at"))
     self.updated_at = dateparser(d.get("updated_at"))
예제 #34
0
def wdmtoswmm5rdii(wdmpath, *dsns, **kwds):
    """Print out DSN data to the screen in SWMM5 RDII format.

    :param wdmpath: Path and WDM filename.
    :param dsns:     The Data Set Numbers in the WDM file.
    :param start_date: If not given defaults to start of data set.
    :param end_date:   If not given defaults to end of data set.
    """
    start_date = kwds.setdefault('start_date', None)
    end_date = kwds.setdefault('end_date', None)

    # Need to make sure that all DSNs are the same interval and all are
    # within start and end dates.
    collect_tcodes = {}
    collect_tsteps = {}
    collect_keys = []
    for dsn in dsns:
        dsn_desc = _describedsn(wdmpath, dsn)
        collect_tcodes[dsn_desc['tcode']] = 1
        collect_tsteps[dsn_desc['tstep']] = 1
        if start_date:
            assert dateparser(start_date) >= dateparser(dsn_desc['start_date'])
        if end_date:
            assert dateparser(end_date) <= dateparser(dsn_desc['end_date'])
        collect_keys.append((dsn_desc['dsn'], dsn_desc['location']))
    assert len(collect_tcodes) == 1
    assert len(collect_tsteps) == 1

    collect_tcodes = list(collect_tcodes.keys())[0]
    collect_tsteps = list(collect_tsteps.keys())[0]

    collected_start_dates = []
    collected_end_dates = []
    collected_ts = {}
    for dsn, location in collect_keys:
        tmp = WDM.read_dsn(wdmpath,
                           int(dsn),
                           start_date=start_date,
                           end_date=end_date)
        collected_start_dates.append(tmp.index[0])
        collected_end_dates.append(tmp.index[-1])
        collected_ts[(dsn, location)] = tmp.values

    maptcode = {
        1: 1,
        2: 60,
        3: 3600,
        4: 86400,
    }

    print('SWMM5')
    print('RDII dump of DSNS {0} from {1}'.format(dsns, wdmpath))
    print(maptcode[collect_tcodes] * collect_tsteps)
    print(1)
    print('FLOW CFS')
    print(len(dsns))
    for dsn, location in collect_keys:
        print(str(dsn) + '_' + location)
    print('Node Year Mon Day Hr Min Sec Flow')
    # Can pick any time series because they should all have the same interval
    # and start and end dates.
    for dex, date in enumerate(tmp.index):
        for dsn, location in collect_keys:
            print('{0}_{1} {2} {3:02} {4:02} {5:02} {6:02} {7:02} {8}'.format(
                dsn, location, date.year, date.month, date.day, date.hour,
                date.minute, date.second, collected_ts[(dsn, location)][dex]))
예제 #35
0
    data_start = invoice_text.find(
        'usage details')  # month of invoice is listed here
    month_str = invoice_text[:data_start].split(' ')[-3:-1]
    month = month_str[0]
    year = month_str[1]
    invoice_text = invoice_text[data_start:]  # no ride data before this

    i = find_next_entry(invoice_text,
                        month)  # find the standard form of a Modo statement

    while i != -1:
        invoice_text = invoice_text[i:]
        month_pos = invoice_text.find(month)
        year_pos = invoice_text.find(year)
        time_pos = invoice_text[year_pos + 4].find(':')
        date = dateparser(invoice_text[month_pos:year_pos + 4] + ' ' +
                          invoice_text[time_pos - 2:time_pos + 3])
        date = date.date()

        split = invoice_text.find('PST') + 12
        if 'PVRT' in invoice_text[split:split + 65]:
            split = invoice_text[split:split + 65].find('PVRT') + 18
        readline = invoice_text[:
                                split]  # collect only the line that this statement is for
        invoice_text = invoice_text[
            split - 3:]  # remove the statement we've already found

        cost = get_money(readline)
        cost = "%.2f" % round(cost, 2)

        if "$" + cost not in search_text:
            print(cost + " seems to be wrong in " + f + " on " + str(date))
예제 #36
0
        # The IHDR chunk must appear FIRST. It contains:
        #   Width:              4 bytes (0:3)
        #   Height:             4 bytes (4:7)
        #   Bit depth:          1 byte (8)
        #   Color type:         1 byte (9)
        #   Compression method: 1 byte (10)
        #   Filter method:      1 byte (11)
        #   Interlace method:   1 byte (12)
        self.header['Width'] = to_int(data[0:4])
        self.header['Height'] = to_int(data[4:8])
        self.header['Bit Depth'] = to_int(data[8])
        self.header['Color Type'] = to_int(data[9])
        self.header['Compression Method'] = to_int(data[10])
        self.header['Filter Method'] = to_int(data[11])
        self.header['Interlace Method'] = to_int(data[12])


if __name__ == '__main__':
    pnginfo = PNGinfo(sys.argv[1])
    text = pnginfo.get_text()
    header = pnginfo.get_header()
    print 'Header:'
    print json.dumps(header, indent=4)
    print 'Comments:'
    for t in text:
        print str(t)
        match = re.search('date:(\D+)([\d\-T\:]+)', t)
        if match:
            timestamp = dateparser(match.group(2))
            print timestamp.astimezone(pytz.utc).isoformat()
예제 #37
0
 def _normalize_datetime_field(self, item: Dict) -> Dict:
     datetime_field = item['properties']['datetime']
     item['properties']['datetime'] = dateparser(datetime_field).strftime(
         DATETIME_FORMAT)
     return item
예제 #38
0
    def handle_gpsposition0(self, topic, body):
        # example: 2:$GPRMC,225030.00,A,3725.1974462,N,12203.8994696,W,,,220216,0.0,E,A*2B

        serverTimestamp = datetime.datetime.now(pytz.utc)

        if body == 'NO DATA':
            logging.info('NO DATA')
            return

        # parse record
        resourceIdStr, trackName, content = body.split(":", 2)
        resourceId = int(resourceIdStr)
        if not checkDataQuality(resourceId, content):
            logging.info('UNRECOGNIZED OR CORRUPT GPS SENTENCE: %s', content)
            return
        sentenceType, utcTime, activeVoid, lat, latHemi, lon,\
            lonHemi, speed, heading, date, declination, declinationDir,\
            modeAndChecksum = content.split(",")
        if OVERRIDE_GPS_DATE:
            serverTime = datetime.datetime.now(pytz.utc)
            overrideDate = serverTime.strftime("%d%m%y")
            sourceTimestamp = datetime.datetime.strptime(
                '%s %s' % (overrideDate, utcTime), '%d%m%y %H%M%S.%f')
        else:
            sourceTimestamp = datetime.datetime.strptime(
                '%s %s' % (date, utcTime), '%d%m%y %H%M%S.%f')
        sourceTimestamp = sourceTimestamp.replace(tzinfo=pytz.utc)
        lat = parseTracLinkDM(lat, latHemi)
        lon = parseTracLinkDM(lon, lonHemi)

        # Get compass heading from compass record
        # TODO this clobbers heading read from GPS every time. but this is for basalt. do we care?
        heading = None
        compassCacheKey = 'compass.%s' % resourceId
        compassInfoString = cache.get(compassCacheKey)
        try:
            if compassInfoString:
                compassInfo = json.loads(compassInfoString)
                compassRecord = compassInfo["compassRecord"]
                # sanity check the timestamp in the compass record
                compassTimeString = compassInfo['timestamp']
                compassTimestamp = dateparser(compassTimeString)
                tdelta = serverTimestamp - compassTimestamp
                if tdelta.total_seconds() <= MAX_COMPASS_TIME_SECONDS:
                    heading = float(compassRecord["compass"])
        except:
            traceback.print_exc()

        # save subsystem status to cache
        myKey = "telemetryCleanup"
        status = {
            'name': myKey,
            'displayName': 'Telemetry Cleanup',
            'statusColor': OKAY_COLOR,
            'lastUpdated': datetime.datetime.utcnow().isoformat(),
            'elapsedTime': ''
        }

        cache.set(myKey, json.dumps(status))

        # calculate which track record belongs to
        cacheKey = 'gpstrack.%s' % resourceId
        pickledTrack = cache.get(cacheKey)
        if pickledTrack:
            # cache hit, great
            track = pickle.loads(pickledTrack)
        else:
            # check db for a track matching this resourceId
            try:
                basaltVehicle = BasaltVehicle.objects.get(
                    resourceId=resourceId)
            except ObjectDoesNotExist:
                logging.warning('%s', traceback.format_exc())
                raise KeyError(
                    'Received GPS position for the EV with resourceId %s. Please ensure there is a vehicle with that id in the BasaltVehicle table.'
                    % resourceId)

            # Check for track name.  We use explicit name if specified, otherwise
            # we check for an active flight and finally use the resourceId
            if len(trackName):
                logging.info("Using track name from listener: %s" % trackName)
            if len(trackName
                   ) == 0:  # I.e. we were not given a name for track already
                try:
                    activeFlight = BasaltActiveFlight.objects.get(
                        flight__vehicle=basaltVehicle)
                    trackName = activeFlight.flight.name
                    logging.info(
                        "Using track name from BasaltActiveFlight: %s" %
                        trackName)
                except ObjectDoesNotExist:
                    trackName = basaltVehicle.name
                    logging.info("Using track name from EV arg: %s" %
                                 trackName)

            tracks = BasaltTrack.objects.filter(name=trackName)
            assert len(tracks) in (0, 1)
            if tracks:
                # we already have a valid track, use that
                track = tracks[0]
            else:
                # must start a new track
                track = BasaltTrack(name=trackName,
                                    vehicle=basaltVehicle,
                                    iconStyle=DEFAULT_ICON_STYLE,
                                    lineStyle=DEFAULT_LINE_STYLE,
                                    dataType=RAW_DATA_TYPE)
                track.save()

            # set cache for next time
            pickledTrack = pickle.dumps(track, pickle.HIGHEST_PROTOCOL)
            cache.set(cacheKey, pickledTrack, TRACK_CACHE_TIMEOUT)

        ######################################################################
        # asset position
        ######################################################################

        # create a NewAssetPosition row
        params = {
            'track': track,
            'timestamp': sourceTimestamp,
            'serverTimestamp': serverTimestamp,
            'latitude': lat,
            'longitude': lon,
            'heading': heading,
            'altitude': None,
        }
        pos = PastPosition(**params)
        pos.save()  # note: could queue for bulk save instead

        cpos = CurrentPosition(**params)
        cpos.saveCurrent()

        pos.broadcast()

        self.publisher.sendDjango(cpos)