Esempio n. 1
0
def get_last_reboot():
    import subprocess
    log = subprocess.getoutput("last reboot --time-format iso")
    newest_boot = log.split("\n")[0].split()[4]
    reboot_time = dateparse(newest_boot)
    if reboot_time.year == 1970:
        newest_halt = log.split("\n")[1].split()[5]
        reboot_time = dateparse(newest_halt)
    return reboot_time
 def map_platform_to_schema(self, concert, band, mbid, other):
     venue = concert["place"]["name"] if "place" in concert else None
     stad = concert["place"]["location"][
         "city"] if "place" in concert and "location" in concert[
             "place"] and "city" in concert["place"]["location"] else None
     state = concert["place"]["location"][
         "state"] if "place" in concert and "location" in concert[
             "place"] and "state" in concert["place"]["location"] else None
     land = concert["place"]["location"][
         "country"] if "place" in concert and "location" in concert[
             "place"] and "country" in concert["place"]["location"] else None
     if state is not None and stad is not None and land in [
             "United States", "Brazil", "Canada", "Australia"
     ]:
         stad = stad + ", " + state
     einddatum = Timestamp(dateparse(
         concert["end_time"]).date()) if "end_time" in concert else None
     return {
         "titel":
         concert["name"] if "name" in concert else None,
         "titel_generated":
         str(band) + " @ " + str(venue) + " in " + str(stad) + ", " +
         str(land),
         "datum":
         Timestamp(dateparse(concert["start_time"]).date()),
         "einddatum":
         einddatum,
         "event_type":
         "festival" if einddatum else None,
         "artiest":
         band,
         "artiest_id":
         "facebook_" + other["page_label"],
         "artiest_mb_naam":
         band,
         "artiest_mb_id":
         mbid,
         "stad":
         stad,
         "land":
         land,
         "venue":
         venue,
         "latitude":
         concert["place"]["location"]["latitude"]
         if "place" in concert and "location" in concert["place"]
         and "latitude" in concert["place"]["location"] else None,
         "longitude":
         concert["place"]["location"]["longitude"]
         if "place" in concert and "location" in concert["place"]
         and "longitude" in concert["place"]["location"] else None,
         "source":
         self.platform,
         "event_id":
         "facebook" + concert["id"]
     }
 def get_event(self, event_id, test_file=None, test=False):
     url = "https://mobile.facebook.com/events/" + str(event_id)
     print(url)
     r = None
     event_data = {}
     if not test:
         r = self.get_html(url)
     else:
         with open(test_file, "r", "utf-8") as f:
             r = f.read()
     if r:
         soup = BeautifulSoup(r.text, 'html.parser')
         try:
             ld = loads(
                 soup.find("script", {
                     "type": "application/ld+json"
                 }).text)
             print("raw data", ld)
             if ld["@type"] == "Event":
                 datum = dateparse(ld["startDate"])
                 einddatum = dateparse(
                     ld["endDate"]) if "endDate" in ld else None
                 if einddatum:
                     if einddatum.hour < 12 and datum < einddatum:
                         einddatum = (einddatum - timedelta(days=1))
                     einddatum = einddatum.date()
                     if einddatum == datum.date():
                         einddatum = None
                 datum = datum.date()
                 location = self._get_location(ld)
                 titel = ld["name"]
                 desc = ld["description"]
                 event_data = {
                     "event_id": event_id,
                     "datum": datum,
                     "einddatum": einddatum,
                     "land": location["country"],
                     "stad": location["city"],
                     "venue": location["venue"],
                     "latitude": location["lat"],
                     "longitude": location["lng"],
                     "titel": titel[0:199],
                     "description": desc
                 }
                 print("event", event_data)
         except AttributeError as e:
             print("error", e)
             event_data = {}
     return event_data
Esempio n. 4
0
 def download_for_date(self, datearg: str, folder: str = './') -> dict:
     dt = dateparse(datearg)
     if dt is None:
         print('the date {} couldnt be parsed'.format(datearg))
         return {}
     else:
         link = list(self.df.loc[self.df['Date'] == dt]['Link'])
         if len(link) == 0:
             print('no record for the date {} found'.format(
                 dt.strftime("%Y/%m/%d")))
             return {}
         else:
             link = link[0]
             Path(folder).mkdir(parents=True, exist_ok=True)
             filename = join(folder, self.get_filename_from_link(link))
             if not exists(filename):
                 with urlopen(link) as response, open(filename,
                                                      'wb') as out_file:
                     copyfileobj(response, out_file)
                     print('report for date {} downloaded at {}'.format(
                         dt.strftime("%Y/%m/%d"), filename))
                     return {'file': filename, 'data': dt}
             else:
                 print(
                     "report for the date {} already exists at {}. didn't re-download"
                     .format(dt.strftime("%Y/%m/%d"), filename))
                 return {'file': filename, 'data': dt}
Esempio n. 5
0
    def _get_event(self, event_id, test_file=None, test=False):
        url = "http://mobile.facebook.com/events/" + event_id
        headers = {'user-agent': self.ua.random}
        r = """<html></html>"""
        if not test:
            sleep(5.0)
            try:
                r = get(url, headers=headers).text
            except Exception as e:
                self._get_event(event_id, test_file, test)
        else:
            with open(test_file, "r", "utf-8") as f:
                r = f.read()
        soup = BeautifulSoup(r, 'html.parser')
        try:
            ld = loads(soup.find("script", {"type": "application/ld+json"}).text)
            if ld["@type"] == "Event":
                datum = dateparse(ld["startDate"])
                einddatum = dateparse(ld["endDate"]) if "endDate" in ld else None
                if einddatum:
                    if einddatum.hour < 12 and datum < einddatum:
                        einddatum = (einddatum - timedelta(days=1))
                    einddatum = einddatum.date()
                    if einddatum == datum.date():
                        einddatum = None
                datum = datum.date()

                location = self._get_location(ld)
                titel = ld["name"]
                cancelled = "Cancel" in ld["eventStatus"] if "eventStatus" in ld else False
                event_data = {
                    "event_id": event_id,
                    "datum": datum,
                    "einddatum": einddatum,
                    "land": location["country"],
                    "stad": location["city"],
                    "venue": location["venue"],
                    "latitude": location["lat"],
                    "longitude": location["lng"],
                    "titel": titel[0:199],
                    "cancelled": cancelled
                }
        except AttributeError as e:
            print("error FB", e)
            event_data = {}
        return event_data
Esempio n. 6
0
 def map_platform_to_schema(self, event, band, mbid, other):
     concertdate = Timestamp(dateparse(event["start"]["date"]).date())
     return {
         "titel":
         event["displayName"].strip().rstrip(
             concertdate.strftime("%B %d, %Y")),
         "titel_generated":
         event["displayName"].strip().rstrip(
             concertdate.strftime("%B %d, %Y")),
         "datum":
         concertdate,
         "einddatum":
         Timestamp(dateparse(event["end"]["date"]).date())
         if "end" in event else None,
         "artiest":
         other["artist_name"],
         "artiest_id":
         "songkick_" + str(other["artist_id"]),
         "artiest_mb_naam":
         band,
         "artiest_mb_id":
         mbid,
         "stad":
         ",".join([
             i.strip() for i in event["location"]["city"].split(",")[0:-1]
         ]),
         "land":
         event["location"]["city"].split(",")[-1].strip(),
         "venue":
         event["displayName"].strip() if event["type"] == "Festival" else
         event["venue"]["displayName"].strip(),
         "latitude":
         event["venue"]["lat"],
         "longitude":
         event["venue"]["lng"],
         "source":
         self.platform,
         "event_id":
         "songkick_" + str(event["id"]),
         "event_type":
         event["type"].lower()
     }
Esempio n. 7
0
 def map_platform_to_schema(self, event, band, mbid, other):
     concertdate = dateparse(event["start"]["date"]).date()
     end_date = dateparse(event["end"]["date"]).date() if "end" in event else None
     is_festival = event["type"] == "Festival"
     return {
         "titel": event["displayName"].strip().rstrip(concertdate.strftime("%B %d, %Y")),
         "datum": concertdate,
         "einddatum": end_date,
         "is_festival": is_festival,
         "artiest": other["artist_name"],
         "artiest_id": str(other["artist_id"]),
         "artiest_mb_naam": band.name,
         "artiest_mb_id": mbid,
         "stad": ",".join([i.strip() for i in event["location"]["city"].split(",")[0:-1]]),
         "land": event["location"]["city"].split(",")[-1].strip(),
         "venue": event["displayName"].strip().replace(str(concertdate.year), "") if event["type"] == "Festival" else event["venue"]["displayName"].strip(),
         "latitude": event["venue"]["lat"],
         "longitude": event["venue"]["lng"],
         "source": self.platform,
         "event_id": str(event["id"]),
         "event_type": event["type"].lower(),
         "cancelled": event["status"] != "ok"
     }
Esempio n. 8
0
 def status(self):
     if self.watcher:
         try:
             d = dateparse(self.watcher.update(),
                           settings={'STRICT_PARSING': True})
             if d:
                 return d
             else:
                 return self.watcher.update()
         except TypeError:
             return None
     else:
         self._logger.error('No watcher associated', exc_info=True)
         return None
Esempio n. 9
0
 def test_condition(self):
     """
     .. deprecated:: 0.1
        Use :func:`eval` instead.
     """
     results = {}
     condition = self.condition.replace('$status', self.watcher.update())
     prefix, comparator, postfix = re.split(r'\s([<>(<=)(>=)(==)(!=)])\s',
                                            condition)
     prefix = "pendulum.instance(dateparse('{}'))".format(
         prefix) if dateparse(prefix, settings={
             'STRICT_PARSING': True
         }) else "'{}'".format(prefix) if (
             isinstance(prefix, str) and not prefix.isnumeric()) else prefix
     postfix = "pendulum.instance(dateparse('{}'))".format(
         postfix) if dateparse(postfix, settings={
             'STRICT_PARSING': True
         }) else "'{}'".format(postfix) if (isinstance(
             postfix, str) and not postfix.isnumeric()) else postfix
     expression = "results['content'] = {} {} {}".format(
         prefix, comparator, postfix)
     exec(expression)
     return results['content']
Esempio n. 10
0
 def __init__(self,
              project_id,
              item_id,
              description,
              priority,
              done,
              due_date=None):
     self.project_id = project_id
     self.id = item_id
     self.description = description
     self.priority = priority
     self.done = done
     if due_date is None:
         self.due_date = None
     else:
         if isinstance(due_date, str):
             self.due_date = dateparse(due_date)
         else:
             self.due_date = due_date
Esempio n. 11
0
def get_date_strict(
        d: Union[datetime, date, str],
        date_locales: Union[Iterable[str], str] = ('ru', 'en'),
        tzinfo: Union[tzinfo, None] = None,
        as_timestamp: bool = False) -> Union[int, float, datetime, date]:
    """ Получить дату из строки, либо вернуть исходную (если это уже объект даты Python) """

    date_locales = list(always_iterable(date_locales))

    result = d if isinstance(d, (datetime, date)) else dateparse(
        d, languages=date_locales, settings={'STRICT_PARSING': True})

    if isinstance(result, datetime):
        result = result.replace(
            tzinfo=tzinfo
        )  ### аргумент <tzinfo> есть только у объектов <datetime>
        result = result.timestamp(
        ) if as_timestamp else result  ### <timestamp> есть только у объектов <datetime>

    return result
Esempio n. 12
0
 def map_platform_to_schema(self, concert, band, mbid, other):
     region = concert["venue"]["region"].strip() if "region" in concert["venue"] else None
     stad = (concert["venue"]["city"]).strip()
     if region is not None and (concert["venue"]["country"]).strip() in ["Ac United States", "United States", "Canada", "Brazil", "Australia"]:
         stad = stad + ", " + region.strip()
     return {
         "datum": dateparse(concert["datetime"]).date(),
         "land": (concert["venue"]["country"]).strip(),
         "stad": stad,
         "venue": (concert["venue"]["name"]).strip() if "name" in concert["venue"] else None,
         "titel": (str(band) + " - " + str((concert["venue"]["name"]).strip()) if "name" in concert["venue"] else "Venue unknown") + "," + dateparse(concert["datetime"]).date().isoformat() + (concert["description"]).strip(),
         "artiest": band,
         "artiest_mb_naam": band,
         "artiest_id": "bandsintown_" + str(concert["artist_id"]),
         "artiest_mb_id": mbid,
         "event_id": str(concert["id"]),
         "latitude": concert["venue"]["latitude"] if "latitude" in concert["venue"] else None,
         "longitude": concert["venue"]["longitude"] if "longitude" in concert["venue"] else None,
         "source": self.platform
     }
 def map_platform_to_schema(self, concert, band, mbid, other):
     stad = concert["venue"]["city"]["name"]
     state = concert["venue"]["city"]["stateCode"] if "stateCode" in concert["venue"]["city"] else None
     if state is not None and concert["venue"]["city"]["country"]["code"] in ["US", "Brazil", "Australia", "Canada"]:
         stad = stad + ", " + state
     return {
         "titel": concert["info"] if "info" in concert else band.name + " @ " + concert["venue"]["name"] + " in " + concert["venue"]["city"]["name"] + ", " + concert["venue"]["city"]["country"]["code"],
         "datum": dateparse(concert["eventDate"], ["%d-%m-%Y"]).date(),
         "artiest": concert["artist"]["name"],
         "artiest_id": concert["artist"]["url"],
         "artiest_mb_naam": band.name,
         "artiest_mb_id": mbid,
         "stad": stad,
         "land": concert["venue"]["city"]["country"]["code"],
         "venue": concert["venue"]["name"],
         "latitude": concert["venue"]["city"]["coords"]["lat"] if "lat" in concert["venue"]["city"]["coords"] else None,
         "longitude": concert["venue"]["city"]["coords"]["long"] if "long" in concert["venue"]["city"]["coords"] else None,
         "source": other,
         "event_id": concert["id"]
     }
def make_date(text):
    try:
        date = dateparse(text)
    except ValueError:
        date = None
    return (text, date)
Esempio n. 15
0
 def _get_datum(ld):
     return dateparse(ld["startDate"]).date()
Esempio n. 16
0
#!/usr/bin/python
from feedparser import parse as feedparse
from dateparser import parse as dateparse

url = "https://www.archlinuxjp.org/feeds/news.xml"
date_fmt = "%Y/%m/%d"

def get_last_reboot():
    import subprocess
    log = subprocess.getoutput("last reboot --time-format iso")
    newest_boot = log.split("\n")[0].split()[4]
    reboot_time = dateparse(newest_boot)
    if reboot_time.year == 1970:
        newest_halt = log.split("\n")[1].split()[5]
        reboot_time = dateparse(newest_halt)
    return reboot_time

last_reboot = get_last_reboot()
responce = feedparse(url)
newer_entries = [f"{dateparse(entry.updated).strftime(date_fmt)} {entry.title}"
                 for entry in responce.entries
                 if dateparse(entry.updated) > last_reboot]

if newer_entries:
    print(responce.feed.title)
    print(*newer_entries, sep="\n")
Esempio n. 17
0
    def handle(self, *args, **options):
        sk_api_key = GigFinder.objects.filter(
            name="www.songkick.com").first().api_key
        platform = GigFinder.objects.filter(name="www.songkick.com").first()

        # loop through all announcements of songkick that are not ignored
        for ca in ConcertAnnouncement.objects.filter(
                gigfinder__name="www.songkick.com").exclude(
                    ignore=True).filter(id__gt=292831).order_by('pk'):

            print("working on", ca, ca.pk)

            # fetch data from songkick
            sk_url = "https://api.songkick.com/api/3.0/events/" + ca.gigfinder_concert_id + ".json?apikey=" + sk_api_key
            print(sk_url)
            time.sleep(0.5)
            html = get(sk_url).text
            data = loads(html) if html is not None else {}
            try:
                result = data["resultsPage"]["results"]["event"]
                #print(dumps(result, indent=4))

                # if is festival or has end date
                if (result["type"] == "Festival"
                        and ca.is_festival != result["type"] == "Festival"
                    ) or ("end" in result and ca.until_date != dateparse(
                        result["end"]["date"]).date()):
                    ca.is_festival = result["type"] == "Festival"
                    ca.until_date = dateparse(result["end"]["date"]).date(
                    ) if "end" in result else None

                    # if is festival and has unknown in venue name, change venue to festival name
                    if result[
                            "type"] == "Festival" and "unknown" in ca.raw_venue.raw_venue.lower(
                            ):
                        venue_name = "|".join([
                            result["displayName"], ",".join(
                                result["location"]["city"].split(",")[0:-1]),
                            result["location"]["city"].split(",")[-1],
                            platform.name
                        ])
                        venue = Venue.objects.filter(
                            raw_venue=venue_name).first()
                        if not venue:
                            venue = Venue.objects.create(
                                raw_venue=venue_name[0:199],
                                raw_location="|".join([
                                    ",".join(result["location"]["city"].split(
                                        ",")[0:-1]),
                                    result["location"]["city"].split(",")[-1],
                                    platform.name
                                ])[0:199])
                            venue.save()

                        # decouple raw venue from organisation
                        print("about to change raw_venue to", venue)
                        ca.raw_venue.organisation = None
                        ca.raw_venue.save()

                        # couple new raw venue to concert announcement
                        ca.raw_venue = venue

                    # update announcement
                    print(
                        "about to change", ca.pk, "and setting is_festival to",
                        result["type"] == "Festival", "until_date to",
                        dateparse(result["end"]["date"]).date()
                        if "end" in result else None)
                    ca.save(update_fields=[
                        'raw_venue', 'until_date', 'is_festival'
                    ])

            except KeyError:
                print(ca.pk, "not available anymore")