def get_last_reboot():
import subprocess
log = subprocess.getoutput("last reboot --time-format iso")
newest_boot = log.split("\n")[0].split()[4]
reboot_time = dateparse(newest_boot)
if reboot_time.year == 1970:
newest_halt = log.split("\n")[1].split()[5]
reboot_time = dateparse(newest_halt)
return reboot_time
def map_platform_to_schema(self, concert, band, mbid, other):
venue = concert["place"]["name"] if "place" in concert else None
stad = concert["place"]["location"][
"city"] if "place" in concert and "location" in concert[
"place"] and "city" in concert["place"]["location"] else None
state = concert["place"]["location"][
"state"] if "place" in concert and "location" in concert[
"place"] and "state" in concert["place"]["location"] else None
land = concert["place"]["location"][
"country"] if "place" in concert and "location" in concert[
"place"] and "country" in concert["place"]["location"] else None
if state is not None and stad is not None and land in [
"United States", "Brazil", "Canada", "Australia"
]:
stad = stad + ", " + state
einddatum = Timestamp(dateparse(
concert["end_time"]).date()) if "end_time" in concert else None
return {
"titel":
concert["name"] if "name" in concert else None,
"titel_generated":
str(band) + " @ " + str(venue) + " in " + str(stad) + ", " +
str(land),
"datum":
Timestamp(dateparse(concert["start_time"]).date()),
"einddatum":
einddatum,
"event_type":
"festival" if einddatum else None,
"artiest":
band,
"artiest_id":
"facebook_" + other["page_label"],
"artiest_mb_naam":
band,
"artiest_mb_id":
mbid,
"stad":
stad,
"land":
land,
"venue":
venue,
"latitude":
concert["place"]["location"]["latitude"]
if "place" in concert and "location" in concert["place"]
and "latitude" in concert["place"]["location"] else None,
"longitude":
concert["place"]["location"]["longitude"]
if "place" in concert and "location" in concert["place"]
and "longitude" in concert["place"]["location"] else None,
"source":
self.platform,
"event_id":
"facebook" + concert["id"]
}
def get_event(self, event_id, test_file=None, test=False):
url = "https://mobile.facebook.com/events/" + str(event_id)
print(url)
r = None
event_data = {}
if not test:
r = self.get_html(url)
else:
with open(test_file, "r", "utf-8") as f:
r = f.read()
if r:
soup = BeautifulSoup(r.text, 'html.parser')
try:
ld = loads(
soup.find("script", {
"type": "application/ld+json"
}).text)
print("raw data", ld)
if ld["@type"] == "Event":
datum = dateparse(ld["startDate"])
einddatum = dateparse(
ld["endDate"]) if "endDate" in ld else None
if einddatum:
if einddatum.hour < 12 and datum < einddatum:
einddatum = (einddatum - timedelta(days=1))
einddatum = einddatum.date()
if einddatum == datum.date():
einddatum = None
datum = datum.date()
location = self._get_location(ld)
titel = ld["name"]
desc = ld["description"]
event_data = {
"event_id": event_id,
"datum": datum,
"einddatum": einddatum,
"land": location["country"],
"stad": location["city"],
"venue": location["venue"],
"latitude": location["lat"],
"longitude": location["lng"],
"titel": titel[0:199],
"description": desc
}
print("event", event_data)
except AttributeError as e:
print("error", e)
event_data = {}
return event_data
def download_for_date(self, datearg: str, folder: str = './') -> dict:
dt = dateparse(datearg)
if dt is None:
print('the date {} couldnt be parsed'.format(datearg))
return {}
else:
link = list(self.df.loc[self.df['Date'] == dt]['Link'])
if len(link) == 0:
print('no record for the date {} found'.format(
dt.strftime("%Y/%m/%d")))
return {}
else:
link = link[0]
Path(folder).mkdir(parents=True, exist_ok=True)
filename = join(folder, self.get_filename_from_link(link))
if not exists(filename):
with urlopen(link) as response, open(filename,
'wb') as out_file:
copyfileobj(response, out_file)
print('report for date {} downloaded at {}'.format(
dt.strftime("%Y/%m/%d"), filename))
return {'file': filename, 'data': dt}
else:
print(
"report for the date {} already exists at {}. didn't re-download"
.format(dt.strftime("%Y/%m/%d"), filename))
return {'file': filename, 'data': dt}
def _get_event(self, event_id, test_file=None, test=False):
url = "http://mobile.facebook.com/events/" + event_id
headers = {'user-agent': self.ua.random}
r = """<html></html>"""
if not test:
sleep(5.0)
try:
r = get(url, headers=headers).text
except Exception as e:
self._get_event(event_id, test_file, test)
else:
with open(test_file, "r", "utf-8") as f:
r = f.read()
soup = BeautifulSoup(r, 'html.parser')
try:
ld = loads(soup.find("script", {"type": "application/ld+json"}).text)
if ld["@type"] == "Event":
datum = dateparse(ld["startDate"])
einddatum = dateparse(ld["endDate"]) if "endDate" in ld else None
if einddatum:
if einddatum.hour < 12 and datum < einddatum:
einddatum = (einddatum - timedelta(days=1))
einddatum = einddatum.date()
if einddatum == datum.date():
einddatum = None
datum = datum.date()
location = self._get_location(ld)
titel = ld["name"]
cancelled = "Cancel" in ld["eventStatus"] if "eventStatus" in ld else False
event_data = {
"event_id": event_id,
"datum": datum,
"einddatum": einddatum,
"land": location["country"],
"stad": location["city"],
"venue": location["venue"],
"latitude": location["lat"],
"longitude": location["lng"],
"titel": titel[0:199],
"cancelled": cancelled
}
except AttributeError as e:
print("error FB", e)
event_data = {}
return event_data
def map_platform_to_schema(self, event, band, mbid, other):
concertdate = Timestamp(dateparse(event["start"]["date"]).date())
return {
"titel":
event["displayName"].strip().rstrip(
concertdate.strftime("%B %d, %Y")),
"titel_generated":
event["displayName"].strip().rstrip(
concertdate.strftime("%B %d, %Y")),
"datum":
concertdate,
"einddatum":
Timestamp(dateparse(event["end"]["date"]).date())
if "end" in event else None,
"artiest":
other["artist_name"],
"artiest_id":
"songkick_" + str(other["artist_id"]),
"artiest_mb_naam":
band,
"artiest_mb_id":
mbid,
"stad":
",".join([
i.strip() for i in event["location"]["city"].split(",")[0:-1]
]),
"land":
event["location"]["city"].split(",")[-1].strip(),
"venue":
event["displayName"].strip() if event["type"] == "Festival" else
event["venue"]["displayName"].strip(),
"latitude":
event["venue"]["lat"],
"longitude":
event["venue"]["lng"],
"source":
self.platform,
"event_id":
"songkick_" + str(event["id"]),
"event_type":
event["type"].lower()
}
def map_platform_to_schema(self, event, band, mbid, other):
concertdate = dateparse(event["start"]["date"]).date()
end_date = dateparse(event["end"]["date"]).date() if "end" in event else None
is_festival = event["type"] == "Festival"
return {
"titel": event["displayName"].strip().rstrip(concertdate.strftime("%B %d, %Y")),
"datum": concertdate,
"einddatum": end_date,
"is_festival": is_festival,
"artiest": other["artist_name"],
"artiest_id": str(other["artist_id"]),
"artiest_mb_naam": band.name,
"artiest_mb_id": mbid,
"stad": ",".join([i.strip() for i in event["location"]["city"].split(",")[0:-1]]),
"land": event["location"]["city"].split(",")[-1].strip(),
"venue": event["displayName"].strip().replace(str(concertdate.year), "") if event["type"] == "Festival" else event["venue"]["displayName"].strip(),
"latitude": event["venue"]["lat"],
"longitude": event["venue"]["lng"],
"source": self.platform,
"event_id": str(event["id"]),
"event_type": event["type"].lower(),
"cancelled": event["status"] != "ok"
}
def status(self):
if self.watcher:
try:
d = dateparse(self.watcher.update(),
settings={'STRICT_PARSING': True})
if d:
return d
else:
return self.watcher.update()
except TypeError:
return None
else:
self._logger.error('No watcher associated', exc_info=True)
return None
def test_condition(self):
"""
.. deprecated:: 0.1
Use :func:`eval` instead.
"""
results = {}
condition = self.condition.replace('$status', self.watcher.update())
prefix, comparator, postfix = re.split(r'\s([<>(<=)(>=)(==)(!=)])\s',
condition)
prefix = "pendulum.instance(dateparse('{}'))".format(
prefix) if dateparse(prefix, settings={
'STRICT_PARSING': True
}) else "'{}'".format(prefix) if (
isinstance(prefix, str) and not prefix.isnumeric()) else prefix
postfix = "pendulum.instance(dateparse('{}'))".format(
postfix) if dateparse(postfix, settings={
'STRICT_PARSING': True
}) else "'{}'".format(postfix) if (isinstance(
postfix, str) and not postfix.isnumeric()) else postfix
expression = "results['content'] = {} {} {}".format(
prefix, comparator, postfix)
exec(expression)
return results['content']
def __init__(self,
project_id,
item_id,
description,
priority,
done,
due_date=None):
self.project_id = project_id
self.id = item_id
self.description = description
self.priority = priority
self.done = done
if due_date is None:
self.due_date = None
else:
if isinstance(due_date, str):
self.due_date = dateparse(due_date)
else:
self.due_date = due_date
def get_date_strict(
d: Union[datetime, date, str],
date_locales: Union[Iterable[str], str] = ('ru', 'en'),
tzinfo: Union[tzinfo, None] = None,
as_timestamp: bool = False) -> Union[int, float, datetime, date]:
""" Получить дату из строки, либо вернуть исходную (если это уже объект даты Python) """
date_locales = list(always_iterable(date_locales))
result = d if isinstance(d, (datetime, date)) else dateparse(
d, languages=date_locales, settings={'STRICT_PARSING': True})
if isinstance(result, datetime):
result = result.replace(
tzinfo=tzinfo
) ### аргумент <tzinfo> есть только у объектов <datetime>
result = result.timestamp(
) if as_timestamp else result ### <timestamp> есть только у объектов <datetime>
return result
def map_platform_to_schema(self, concert, band, mbid, other):
region = concert["venue"]["region"].strip() if "region" in concert["venue"] else None
stad = (concert["venue"]["city"]).strip()
if region is not None and (concert["venue"]["country"]).strip() in ["Ac United States", "United States", "Canada", "Brazil", "Australia"]:
stad = stad + ", " + region.strip()
return {
"datum": dateparse(concert["datetime"]).date(),
"land": (concert["venue"]["country"]).strip(),
"stad": stad,
"venue": (concert["venue"]["name"]).strip() if "name" in concert["venue"] else None,
"titel": (str(band) + " - " + str((concert["venue"]["name"]).strip()) if "name" in concert["venue"] else "Venue unknown") + "," + dateparse(concert["datetime"]).date().isoformat() + (concert["description"]).strip(),
"artiest": band,
"artiest_mb_naam": band,
"artiest_id": "bandsintown_" + str(concert["artist_id"]),
"artiest_mb_id": mbid,
"event_id": str(concert["id"]),
"latitude": concert["venue"]["latitude"] if "latitude" in concert["venue"] else None,
"longitude": concert["venue"]["longitude"] if "longitude" in concert["venue"] else None,
"source": self.platform
}
def map_platform_to_schema(self, concert, band, mbid, other):
stad = concert["venue"]["city"]["name"]
state = concert["venue"]["city"]["stateCode"] if "stateCode" in concert["venue"]["city"] else None
if state is not None and concert["venue"]["city"]["country"]["code"] in ["US", "Brazil", "Australia", "Canada"]:
stad = stad + ", " + state
return {
"titel": concert["info"] if "info" in concert else band.name + " @ " + concert["venue"]["name"] + " in " + concert["venue"]["city"]["name"] + ", " + concert["venue"]["city"]["country"]["code"],
"datum": dateparse(concert["eventDate"], ["%d-%m-%Y"]).date(),
"artiest": concert["artist"]["name"],
"artiest_id": concert["artist"]["url"],
"artiest_mb_naam": band.name,
"artiest_mb_id": mbid,
"stad": stad,
"land": concert["venue"]["city"]["country"]["code"],
"venue": concert["venue"]["name"],
"latitude": concert["venue"]["city"]["coords"]["lat"] if "lat" in concert["venue"]["city"]["coords"] else None,
"longitude": concert["venue"]["city"]["coords"]["long"] if "long" in concert["venue"]["city"]["coords"] else None,
"source": other,
"event_id": concert["id"]
}
def make_date(text):
try:
date = dateparse(text)
except ValueError:
date = None
return (text, date)
def _get_datum(ld):
return dateparse(ld["startDate"]).date()
#!/usr/bin/python
from feedparser import parse as feedparse
from dateparser import parse as dateparse
url = "https://www.archlinuxjp.org/feeds/news.xml"
date_fmt = "%Y/%m/%d"
def get_last_reboot():
import subprocess
log = subprocess.getoutput("last reboot --time-format iso")
newest_boot = log.split("\n")[0].split()[4]
reboot_time = dateparse(newest_boot)
if reboot_time.year == 1970:
newest_halt = log.split("\n")[1].split()[5]
reboot_time = dateparse(newest_halt)
return reboot_time
last_reboot = get_last_reboot()
responce = feedparse(url)
newer_entries = [f"{dateparse(entry.updated).strftime(date_fmt)} {entry.title}"
for entry in responce.entries
if dateparse(entry.updated) > last_reboot]
if newer_entries:
print(responce.feed.title)
print(*newer_entries, sep="\n")
def handle(self, *args, **options):
sk_api_key = GigFinder.objects.filter(
name="www.songkick.com").first().api_key
platform = GigFinder.objects.filter(name="www.songkick.com").first()
# loop through all announcements of songkick that are not ignored
for ca in ConcertAnnouncement.objects.filter(
gigfinder__name="www.songkick.com").exclude(
ignore=True).filter(id__gt=292831).order_by('pk'):
print("working on", ca, ca.pk)
# fetch data from songkick
sk_url = "https://api.songkick.com/api/3.0/events/" + ca.gigfinder_concert_id + ".json?apikey=" + sk_api_key
print(sk_url)
time.sleep(0.5)
html = get(sk_url).text
data = loads(html) if html is not None else {}
try:
result = data["resultsPage"]["results"]["event"]
#print(dumps(result, indent=4))
# if is festival or has end date
if (result["type"] == "Festival"
and ca.is_festival != result["type"] == "Festival"
) or ("end" in result and ca.until_date != dateparse(
result["end"]["date"]).date()):
ca.is_festival = result["type"] == "Festival"
ca.until_date = dateparse(result["end"]["date"]).date(
) if "end" in result else None
# if is festival and has unknown in venue name, change venue to festival name
if result[
"type"] == "Festival" and "unknown" in ca.raw_venue.raw_venue.lower(
):
venue_name = "|".join([
result["displayName"], ",".join(
result["location"]["city"].split(",")[0:-1]),
result["location"]["city"].split(",")[-1],
platform.name
])
venue = Venue.objects.filter(
raw_venue=venue_name).first()
if not venue:
venue = Venue.objects.create(
raw_venue=venue_name[0:199],
raw_location="|".join([
",".join(result["location"]["city"].split(
",")[0:-1]),
result["location"]["city"].split(",")[-1],
platform.name
])[0:199])
venue.save()
# decouple raw venue from organisation
print("about to change raw_venue to", venue)
ca.raw_venue.organisation = None
ca.raw_venue.save()
# couple new raw venue to concert announcement
ca.raw_venue = venue
# update announcement
print(
"about to change", ca.pk, "and setting is_festival to",
result["type"] == "Festival", "until_date to",
dateparse(result["end"]["date"]).date()
if "end" in result else None)
ca.save(update_fields=[
'raw_venue', 'until_date', 'is_festival'
])
except KeyError:
print(ca.pk, "not available anymore")