def get_flights(url: str, database: list):
# get source code of search results on azair.eu
source_code = requests.get(url)
soup = bs4.BeautifulSoup(source_code.text, 'lxml')
# loop through all found flights
flights = soup.findAll('div', {'class': 'result'})
for flight in flights:
try:
# get data of flight there
flight_there = flight.div.findAll('p', recursive=False)[0]
ft_weekday = flight_there.find('span', {'class': 'date'}).text.split(' ')[0]
ft_date = convert_date(flight_there.find('span', {'class': 'date'}).text.split(' ')[1])
ft_from_time = flight_there.find('span', {'class': 'from'}).text.split(' ')[0]
ft_from_city_name = flight_there.find('span', {'class': 'from'}).text.split(' ')[1]
ft_from_city_code = flight_there.find('span', {'class': 'from'}).text.split(' ')[2][:3]
ft_to_time = flight_there.find('span', {'class': 'to'}).text.split(' ')[0]
ft_to_city_name = flight_there.find('span', {'class': 'to'}).text.split(' ')[1]
ft_to_city_code = flight_there.find('span', {'class': 'to'}).text.split(' ')[2][:3]
ft_duration = take_out_time(flight_there.find('span', {'class': 'durcha'}).text.split('/')[0])
ft_changes = take_out_number(flight_there.find('span', {'class': 'durcha'}).text.split('/')[1].strip())
ft_price = take_out_number(flight_there.find('span', {'class': 'subPrice'}).text)
# get data of flight back
flight_back = flight.div.findAll('p', recursive=False)[1]
fb_weekday = flight_back.find('span', {'class': 'date'}).text.split(' ')[0]
fb_date = convert_date(flight_back.find('span', {'class': 'date'}).text.split(' ')[1])
fb_from_time = flight_back.find('span', {'class': 'from'}).text.split(' ')[0]
fb_from_city_name = flight_back.find('span', {'class': 'from'}).text.split(' ')[1]
fb_from_city_code = flight_back.find('span', {'class': 'from'}).text.split(' ')[2][:3]
fb_to_time = flight_back.find('span', {'class': 'to'}).text.split(' ')[0]
fb_to_city_name = flight_back.find('span', {'class': 'to'}).text.split(' ')[1]
fb_to_city_code = flight_back.find('span', {'class': 'to'}).text.split(' ')[2][:3]
fb_duration = take_out_time(flight_back.find('span', {'class': 'durcha'}).text.split('/')[0])
fb_changes = take_out_number(flight_back.find('span', {'class': 'durcha'}).text.split('/')[1].strip())
fb_price = take_out_number(flight_back.find('span', {'class': 'subPrice'}).text)
price = take_out_number(flight.find('span', {'class': 'tp'}).text)
stay_days = take_out_number(flight.find('span', {'class': 'lengthOfStay'}).text)
link = 'http://www.azair.eu/' + flight.find('div', {'class': 'bookmark'}).a.get('href')
result = [ft_weekday, ft_date, ft_from_time, ft_from_city_name, ft_from_city_code, ft_to_time, ft_to_city_name,
ft_to_city_code, ft_duration, ft_changes, ft_price, fb_weekday, fb_date, fb_from_time,
fb_from_city_name, fb_from_city_code, fb_to_time, fb_to_city_name, fb_to_city_code, fb_duration,
fb_changes, fb_price, price, number_people, stay_days, link]
print(result)
database.append(result)
except:
traceback.print_exc()
with open('assets/csv/azair.csv', 'w', newline='', encoding='UTF-8') as fp:
myFile = csv.writer(fp)
myFile.writerows(database)
def add_period(self, period):
start_day = convert_date(period['startmoment'])
end_day = convert_date(period['finishmoment'])
if end_day < self.start_datetime or start_day > self.end_datetime:
return
if start_day < self.start_datetime:
start_day = self.start_datetime
if end_day > self.end_datetime:
end_day = self.end_datetime
for halfday in xhalfdays(start_day, end_day):
self.days[halfday] = PeriodActivity(
activity_type=period['type'],
verbose_type=self.period_legend.get(period['type']),
label=period.get('label')
)
def search(self, results, media, lang, manual):
PlexLog.debug("=================== Search Start ===================")
PlexLog.debug("%s (%s)" % (self.name, self.ver))
PlexLog.debug("Plex version: %s" % Platform.ServerVersion)
server = Prefs["Server"]
if not server:
PlexLog.error("Missing server!")
return
movie = request_json(urljoin(server, "movie"), as_movie(media))
if movie is None:
return
title = movie.get("title")
if title is None:
PlexLog.error("Missing or invalid title: %s" % str(movie))
return
aired = convert_date(movie.get("aired"))
year = aired.year if aired is not None else 0
# Plex throws exception that have "/" in ID
mid = create_id(title, year)
result = MetadataSearchResult(id=mid,
name=title,
year=year,
lang=lang,
score=100)
results.Append(result)
PlexLog.debug("=================== Search end ===================")
def reddit_set_addon_setting_from_response(response):
import time, json
from utils import convert_date
global reddit_access_token #specify "global" if you want to change the value of a global variable
global reddit_refresh_token
try:
response = json.loads(response.replace('\\"', '\''))
log( json.dumps(response, indent=4) )
if 'error' in response:
return response['error']
else:
if 'refresh_token' in response: #refresh_token only returned when getting reddit_get_refresh_token. it is a one-time step
reddit_refresh_token = response['refresh_token']
addon.setSetting('reddit_refresh_token', reddit_refresh_token)
reddit_access_token = response['access_token']
addon.setSetting('reddit_access_token', reddit_access_token)
addon.setSetting('reddit_access_token_scope', response['scope'])
unix_time_now = int(time.time())
unix_time_now += int( response['expires_in'] )
addon.setSetting('reddit_access_token_expires', convert_date(unix_time_now))
except Exception as e:
log(" parsing reddit token response EXCEPTION:="+ str( sys.exc_info()[0]) + " " + str(e) )
return str(e)
return "ok"
def add_period(self, period):
start_day = convert_date(period['startmoment'])
end_day = convert_date(period['finishmoment'])
if end_day < self.start_datetime or start_day > self.end_datetime:
return
if start_day < self.start_datetime:
start_day = self.start_datetime
if end_day > self.end_datetime:
end_day = self.end_datetime
for halfday in xhalfdays(start_day, end_day):
self.days[halfday] = {
'nature': 'period',
'type': period['type'],
'verbose_type': self.period_legend.get(period['type']),
'label': period.get('label'),
}
def set_movie(metadata, movie):
metadata.title = movie.get("title")
metadata.title_sort = movie.get("sort_title")
metadata.original_title = movie.get("original_title")
metadata.content_rating = movie.get("content_rating")
metadata.studio = first_or(movie.get("studio"))
aired = convert_date(movie.get("aired"))
metadata.originally_available_at = aired
metadata.year = aired.year if aired is not None else None
metadata.tagline = " / ".join(movie.get("tagline"))
metadata.summary = movie.get("summary")
metadata.rating = movie.get("rating")
set_metadata_list(metadata, "genres", movie.get("genres"))
set_metadata_list(metadata, "collections", movie.get("collections"))
set_metadata_actors(metadata, movie.get("actors"))
writers = [
w.get("name") for w in movie.get("writers", [])
if w.get("name") is not None
]
directors = [
d.get("name") for d in movie.get("directors", [])
if d.get("name") is not None
]
set_metadata_list_name(metadata, "writers", writers)
set_metadata_list_name(metadata, "directors", directors)
def calc(self, value, day):
if (value > self.last_peak):
self.last_peak = value
self.peak_date = day
elif (value < self.last_peak):
last_day = convert_date(self.peak_date, time=self.time)
new_day = convert_date(day, time=self.time)
if (last_day == ""):
self.max_time = max(self.max_time, 0)
else:
self.max_time = max(self.max_time, (new_day - last_day).days)
if (self.max_height < self.last_peak - value):
self.max_height = self.last_peak - value
def search(self, results, media, lang, manual):
PlexLog.debug("=================== Search Start ===================")
PlexLog.debug("%s (%s)" % (self.name, self.ver))
PlexLog.debug("Plex version: %s" % Platform.ServerVersion)
show = get_show(media)
if show is None:
return
title = show.get("title")
if title is None:
PlexLog.error("Missing or invalid title: %s" % str(show))
return
aired = convert_date(show.get("aired"))
year = aired.year if aired is not None else 0
# Plex throws exception that have "/" in ID
mid = create_id(title, year)
result = MetadataSearchResult(id=mid,
name=title,
year=year,
lang=lang,
score=100)
results.Append(result)
PlexLog.debug("=================== Search end ===================")
def set_episode(metadata, episode):
metadata.title = join_list_or(episode.get("title"))
metadata.content_rating = episode.get("content_rating")
metadata.originally_available_at = convert_date(episode.get("aired"))
metadata.summary = episode.get("summary")
metadata.rating = episode.get("rating")
set_metadata_list_name(metadata, "writers", episode.get("writers", []))
set_metadata_list_name(metadata, "directors", episode.get("directors", []))
def add_symbol(self, symbol):
day = convert_date(symbol['moment'])
if day < self.start_datetime or day > self.end_datetime:
return
for halfday in xhalfdays_for_day(day):
self.days[halfday] = SymbolActivity(
activity_type=symbol['type'],
verbose_type=self.symbol_legend.get(symbol['type'])
)
def set_episode(metadata, episode):
metadata.title = episode.get("title")
metadata.content_rating = episode.get("content_rating")
metadata.originally_available_at = convert_date(episode.get("upload_date"))
metadata.summary = episode.get("description")
average_rating = episode.get("average_rating")
metadata.rating = average_rating * 2 if average_rating is not None else None
set_metadata_list(metadata, "writers", episode.get("writers", []))
set_metadata_list(metadata, "directors", episode.get("directors", []))
def add_symbol(self, symbol):
day = convert_date(symbol['moment'])
if day < self.start_datetime or day > self.end_datetime:
return
for halfday in xhalfdays_for_day(day):
self.days[halfday] = {
'nature': 'symbol',
'type': symbol['type'],
'verbose_type': self.symbol_legend.get(symbol['type']),
'label': self.symbol_legend.get(symbol['type']),
}
def set_show(metadata, channel):
metadata.title = channel.get("title")
metadata.title_sort = channel.get("title")
metadata.original_title = channel.get("title")
metadata.content_rating = channel.get("content_rating")
metadata.studio = channel.get("studio")
metadata.originally_available_at = convert_date(channel.get("aired"))
metadata.summary = channel.get("description")
metadata.rating = channel.get("rating")
set_metadata_list(metadata, "genres", channel.get("keywords").split())
set_metadata_list(metadata, "collections", channel.get("collections", []))
set_metadata_actors(metadata, channel.get("actors", []))
def sendConfirmationEmail(item_id):
# don't even attempt if email config info has not been provided
if not web.config.smtp_username:
print "WARNING: email user/password not configured. Ignoring attempt to send confirmation email."
return
item = model.get_item(int(item_id))
# Get contact info for the help item
post_data = model.get_post(int(item.helpRequestId))
contact_data = model.get_contact_data(post_data.contactId)
item.contactName = contact_data.name
item.contactEmail = contact_data.email
item.contactPhone = contact_data.phone
item.url = "%s/view/%s" % (config.SITE_BASE, item.helpRequestId)
item.date = utils.convert_date(item.date)
if item.helpEmail:
f = web.config.smtp_username
to = item.helpEmail
cc = item.contactEmail
subject = 'Confirmation: you signed up to help on %(date)s' % item
msg = """Thanks for you willingness to help.
This email is to confirm that you signed up to help on %(date)s for the item "%(description)s". More details can be found here:
%(url)s
If you have any questions don't reply to this email. Instead contact %(contactName)s
email: %(contactEmail)s
phone: %(contactPhone)s
Thanks!""" % item
#print f, to, subject, msg
web.sendmail(f,to,subject,msg,cc=cc)
else:
# The person did not provide an email. Instead an email will be sent
# to the contact person informing them of the situation.
f = web.config.smtp_username
to = item.contactEmail
subject = 'Confirmation: Helper with no email'
msg = """%(helpName)s signed up to help on %(date)s for item
"%(description)s" but did not provide an email address.
Since no email is present, the website will be unable to provide them an automatic reminder.
Details can be found here:
%(url)s
""" % item
#print f, to, subject, msg
web.sendmail(f,to,subject,msg)
def sendConfirmationEmail(item_id):
# don't even attempt if email config info has not been provided
if not web.config.smtp_username:
print("WARNING: email user/password not configured. Ignoring attempt to send confirmation email.")
return
item = model.get_item(int(item_id))
# Get contact info for the help item
post_data = model.get_post(int(item.helpRequestId))
contact_data = model.get_contact_data(post_data.contactId)
item.contactName = contact_data.name
item.contactEmail = contact_data.email
item.contactPhone = contact_data.phone
item.url = "%s/view/%s" % (config.SITE_BASE, item.helpRequestId)
item.date = utils.convert_date(item.date)
if item.helpEmail:
f = web.config.smtp_username
to = item.helpEmail
cc = item.contactEmail
subject = 'Confirmation: you signed up to help on %(date)s' % item
msg = """Thanks for you willingness to help.
This email is to confirm that you signed up to help on %(date)s for the item "%(description)s". More details can be found here:
%(url)s
If you have any questions don't reply to this email. Instead contact %(contactName)s
email: %(contactEmail)s
phone: %(contactPhone)s
Thanks!""" % item
#print f, to, subject, msg
web.sendmail(f,to,subject,msg,cc=cc)
else:
# The person did not provide an email. Instead an email will be sent
# to the contact person informing them of the situation.
f = web.config.smtp_username
to = item.contactEmail
subject = 'Confirmation: Helper with no email'
msg = """%(helpName)s signed up to help on %(date)s for item
"%(description)s" but did not provide an email address.
Since no email is present, the website will be unable to provide them an automatic reminder.
Details can be found here:
%(url)s
""" % item
#print f, to, subject, msg
web.sendmail(f,to,subject,msg)
def main(switch_input):
apikey = getenv('APIKEY_OPENWEATHERMAP')
current_weather_url = f"http://api.openweathermap.org/data/2.5/weather?q={switch_input[0].strip(' ')}&units=imperial&appid={apikey}"
response = requests.get(current_weather_url)
weather_table = {}
json_data = response.json()
try:
weather = json_data["weather"]
except TypeError:
print(f"typeError Exception: cannot find weather data in {weather}")
return "TypeError Exception: Cannot find weather data"
main_weather = json_data["main"]
weather_table['main condition'] = weather[0]["main"]
weather_table['condition description'] = weather[0]["description"]
weather_table['temp'] = main_weather["temp"]
weather_table['feels like'] = main_weather["feels_like"]
weather_table['pressure'] = main_weather["pressure"]
weather_table['humidity'] = main_weather["humidity"]
weather_table['city'] = json_data["name"]
weather_table['country'] = json_data["sys"]["country"]
weather_table['wind'] = json_data["wind"]["speed"]
weather_table['latitude'] = json_data["coord"]["lat"]
weather_table['longitude'] = json_data["coord"]["lon"]
weather_table[
'coord'] = f"{weather_table['latitude']}|{weather_table['longitude']}"
out = [f"{item}: {weather_table[item]}" for item in weather_table]
if 'forecast' in switch_input:
try:
one_call_weather_url = f"https://api.openweathermap.org/data/2.5/onecall?lat={weather_table['latitude']}&lon={weather_table['longitude']}&exclude=minutely,hourly&units=imperial&appid={apikey}"
one_call_response = requests.get(one_call_weather_url)
one_call_json = one_call_response.json()
daily_weather_string = "forecast: "
daily = one_call_json["daily"]
for json_day in daily:
unix_date = json_day["dt"]
readable_date = convert_date(unix_date)
daily_weather_string = f"{daily_weather_string}|{readable_date}"
except KeyError:
return "Key Exception: Can't find key Daily"
print(daily_weather_string)
return daily_weather_string
else:
print(out)
return out
def set_album(metadata, media, album):
metadata.summary = album.get("summary")
metadata.rating = album.get("rating")
set_metadata_list(metadata, "genres", album.get("genres"))
set_metadata_list(metadata, "collections", album.get("collections"))
aired = convert_date(album.get("aired"))
metadata.originally_available_at = aired
update_album(media.id, media.title, album.get("collections"))
set_album_cover(metadata, album)
for track in media.children:
part = track.items[0].parts[0].file
update_tracks(track.id, part)
def set_episode(metadata, episode):
metadata.title = join_list_or(episode.get("title"))
metadata.content_rating = episode.get("content_rating")
metadata.originally_available_at = convert_date(episode.get("aired"))
metadata.summary = episode.get("summary")
metadata.rating = episode.get("rating")
writers = [
w.get("name") for w in episode.get("writers", [])
if w.get("name") is not None
]
directors = [
d.get("name") for d in episode.get("directors", [])
if d.get("name") is not None
]
set_metadata_list_name(metadata, "writers", writers)
set_metadata_list_name(metadata, "directors", directors)
def set_show(metadata, media, show):
metadata.title = show.get("title")
metadata.title_sort = show.get("sort_title")
metadata.original_title = show.get("original_title")
metadata.content_rating = show.get("content_rating")
metadata.studio = first_or(show.get("studio"))
metadata.originally_available_at = convert_date(show.get("aired"))
metadata.summary = show.get("summary")
metadata.rating = show.get("rating")
set_metadata_list(metadata, "genres", show.get("genres"))
set_metadata_list(metadata, "collections", show.get("collections"))
set_metadata_actors(metadata, show.get("actors"))
original_title = show.get("original_title")
tagline = join_list_or(show.get("tagline"))
update_show(media.id, original_title, tagline)
def user_entering_birthday(message):
# Проверка на дату
if utils.validate_date(message.text):
bot.send_message(message.chat.id, messages.s_birth_error)
return
birthday = utils.convert_date(message.text)
current_date = datetime.datetime.now()
if (current_date.year - birthday.year) > 100 or (current_date.year - birthday.year) < 2:
bot.send_message(message.chat.id, messages.s_birth_cheat)
return
else:
# Дата рождения введена корректно, можно идти дальше
db.add_user_birth(message.chat.id, birthday) # Добавляем ДР в базу
utils.validate_ref(db, bot, message.chat.id) # Дарим подарок
bot.send_message(message.chat.id, messages.s_menu, reply_markup=markups.keyboardMain)
def add_person(name, gender, title='', dob=None, image=None, summary=''):
person, _ = Person.objects.get_or_create(slug=slugify(name))
person.legal_name = name
person.gender = gender.lower() or 'm'
person.title = title
if dob:
dob = convert_date(dob)
# TODO: investigate using a datetime.date object
person.date_of_birth = ApproximateDate(year=dob.year, month=dob.month, day=dob.day)
if summary:
person.summary = summary
person.can_be_featured = True
if image:
add_person_image(person, base64.decodestring(image))
person.save()
return person
def search(self, results, media, lang, manual):
PlexLog.debug("=================== Search Start ===================")
PlexLog.debug("%s (%s)" % (self.name, self.ver))
PlexLog.debug("Plex version: %s" % Platform.ServerVersion)
server = Prefs["Server"]
authKey = Prefs["AuthKey"]
if not server:
PlexLog.error("Missing server!")
return
requestUrl = urljoin(server, "show")
if authKey:
requestUrl = requestUrl + "?AuthKey=" + authKey
PlexLog.debug("Requesting URL: %s" % requestUrl)
show = request_json(requestUrl, as_show(media))
if show is None:
return
title = show.get("title")
if title is None:
PlexLog.error("Missing or invalid title: %s" % str(show))
return
aired = convert_date(show.get("aired"))
year = aired.year if aired is not None else 0
# Plex throws exception that have "/" in ID
mid = create_id(title, year)
result = MetadataSearchResult(id=mid,
name=title,
year=year,
lang=lang,
score=100)
results.Append(result)
PlexLog.debug("=================== Search end ===================")
def reddit_set_addon_setting_from_response(response):
import time, json
from utils import convert_date
global reddit_access_token #specify "global" if you want to change the value of a global variable
global reddit_refresh_token
try:
response = json.loads(response.replace('\\"', '\''))
log(json.dumps(response, indent=4))
if 'error' in response:
#Error Cause Resolution
#401 response Client credentials sent as HTTP Basic Authorization were invalid Verify that you are properly sending HTTP Basic Authorization headers and that your credentials are correct
#unsupported_grant_type grant_type parameter was invalid or Http Content type was not set correctly Verify that the grant_type sent is supported and make sure the content type of the http message is set to application/x-www-form-urlencoded
#NO_TEXT for field code You didn't include the code parameter Include the code parameter in the POST data
#invalid_grant The code has expired or already been used Ensure that you are not attempting to re-use old codes - they are one time use.
return response['error']
else:
if 'refresh_token' in response: #refresh_token only returned when getting reddit_get_refresh_token. it is a one-time step
reddit_refresh_token = response['refresh_token']
addon.setSetting('reddit_refresh_token', reddit_refresh_token)
reddit_access_token = response['access_token']
addon.setSetting('reddit_access_token', reddit_access_token)
#log( ' new access token '+ reddit_access_token )
addon.setSetting('reddit_access_token_scope', response['scope'])
unix_time_now = int(time.time())
unix_time_now += int(response['expires_in'])
addon.setSetting('reddit_access_token_expires',
convert_date(unix_time_now))
except Exception as e:
log(" parsing reddit token response EXCEPTION:=" +
str(sys.exc_info()[0]) + " " + str(e))
return str(e)
return "ok"
def add_person(name, gender, title='', dob=None, image=None, summary=''):
person, _ = Person.objects.get_or_create(slug=slugify(name))
person.legal_name = name
person.gender = gender.lower() or 'm'
person.title = title
if dob:
dob = convert_date(dob)
# TODO: investigate using a datetime.date object
person.date_of_birth = ApproximateDate(year=dob.year,
month=dob.month,
day=dob.day)
if summary:
person.summary = summary
person.can_be_featured = True
if image:
add_person_image(person, base64.decodestring(image))
person.save()
return person
def reddit_set_addon_setting_from_response(response):
import time, json
from utils import convert_date
global reddit_access_token #specify "global" if you want to change the value of a global variable
global reddit_refresh_token
try:
response = json.loads(response.replace('\\"', '\''))
log( json.dumps(response, indent=4) )
if 'error' in response:
#Error Cause Resolution
#401 response Client credentials sent as HTTP Basic Authorization were invalid Verify that you are properly sending HTTP Basic Authorization headers and that your credentials are correct
#unsupported_grant_type grant_type parameter was invalid or Http Content type was not set correctly Verify that the grant_type sent is supported and make sure the content type of the http message is set to application/x-www-form-urlencoded
#NO_TEXT for field code You didn't include the code parameter Include the code parameter in the POST data
#invalid_grant The code has expired or already been used Ensure that you are not attempting to re-use old codes - they are one time use.
return response['error']
else:
if 'refresh_token' in response: #refresh_token only returned when getting reddit_get_refresh_token. it is a one-time step
reddit_refresh_token = response['refresh_token']
addon.setSetting('reddit_refresh_token', reddit_refresh_token)
reddit_access_token = response['access_token']
addon.setSetting('reddit_access_token', reddit_access_token)
#log( ' new access token '+ reddit_access_token )
addon.setSetting('reddit_access_token_scope', response['scope'])
unix_time_now = int(time.time())
unix_time_now += int( response['expires_in'] )
addon.setSetting('reddit_access_token_expires', convert_date(unix_time_now))
except Exception as e:
log(" parsing reddit token response EXCEPTION:="+ str( sys.exc_info()[0]) + " " + str(e) )
return str(e)
return "ok"
while len(sys.argv) > 1 and sys.argv[1].startswith('-'):
if len(sys.argv) > 1 and sys.argv[1] == "--dry-run":
DRYRUN = True
del sys.argv[1]
elif len(sys.argv) > 2 and sys.argv[1] == '-o':
OUTPUT_DIR = sys.argv[2]
sys.argv[1:3] = []
else:
sys.stderr.write('Unknown switch: ' + sys.argv[1] + '\n')
sys.exit(1)
if len(sys.argv) != 3:
sys.stderr.write("Usage: %s [--dry-run] [-o OUTPUT_DIR] START_DATE END_DATE\n" % sys.argv[0])
sys.exit(1)
start_date = utils.convert_date(sys.argv[1])
end_date = utils.convert_date(sys.argv[2])
if start_date > end_date:
sys.stderr.write("START_DATE is later than END_DATE\n")
sys.exit(1)
if os.path.exists(OUTPUT_DIR):
if not os.path.isdir(OUTPUT_DIR):
sys.stderr.write(OUTPUT_DIR + " is not a directory.\n")
sys.exit(1)
else:
os.makedirs(OUTPUT_DIR)
date = start_date
while date <= end_date:
events = wikipydia.query_current_events(date)
def fieldval_to_id_in_db(self, date):
''' expects a datetime object, returns db id '''
return convert_date(date)
def post(self, node_type):
ret = {'code': 0, 'result': [], 'message': ''}
es_index = "relays"
columns = ts.RELAY_FIELDS
extras = dict()
if 'bridges' not in node_type and 'relays' not in node_type:
ret['code'] = 1
ret['message'] = "Invalid node type"
self.write(ret)
return
try:
body_data = escape.json_decode(self.request.body)
except Exception:
body_data = None
if body_data:
time_type = body_data['vtime']
else:
time_type = self.get_argument('vtime', None)
if time_type is None:
time_type = "first_seen"
elif time_type not in ["first_seen", "last_seen", "last_restarted"]:
ret['code'] = 1
ret['message'] = "Invalid time type"
self.write(ret)
return
if body_data:
daterange = body_data['time']
query = body_data['query']
else:
daterange = self.get_argument('time', None)
query = self.get_argument('query', None)
if not daterange:
ret['code'] = 1
ret['message'] = "Invalid POST data"
self.write(ret)
return
# the date is in format m/d/yyyy - m/d/yyyy
daterange = "".join(daterange.split())
daterange = daterange.split('-')
if not utils.validate_daterange(daterange):
ret['code'] = 1
ret['message'] = "Invalid time range"
self.write(ret)
return
daterange[0] = utils.convert_date(daterange[0])
daterange[1] = utils.convert_date(daterange[1])
extras['time'] = {'type': time_type, 'range': daterange}
if query and not utils.validate_query(query):
ret['code'] = 1
ret['message'] = "Invalid query"
self.write(ret)
return
if body_data:
country = body_data['country']
flags = body_data['flags']
else:
country = self.get_argument('country', None)
flags = self.get_argument('flags', None)
if country is not None and country.strip():
# sanitize input to avoid weird values
if utils.check_extras(country):
extras['country'] = country
if flags is not None and flags.strip():
# sanitize input to avoid weird values
if utils.check_extras(flags):
extras['flags'] = flags
if node_type == "bridges":
es_index = "bridges"
columns = ts.BRIDGE_FIELDS
try:
results = yield self.application.es_instance.search(es_index, query, extras)
except Exception as e:
print traceback.format_exc()
ret['code'] = 1
ret['message'] = "Invalid query"
self.write(ret)
return
for entry in results:
entry['ip_address'] = utils.extract_ipv4(entry)
entry['bandwidth'] = utils.calculate_bandwidth(entry, "advertised_bandwidth")
ret['result'] = {"data": results, "columns": columns}
self.write(ret)
def get_dates(job, req):
date1 = convert_date(req['date1_' + job], req.get('time1_' + job, None))
date2 = convert_date(req.get('date2_' + job, None),
req.get('time2_' + job, None))
return [date1, date2]
# print item.date,t_delta
if timedelta(days=0) < t_delta < timedelta(days=1):
# print 'SEND EMAIL:',item.date,t_delta
tomorrow_items.append(item)
# For each item, if a helper has signed up, send an email reminder (using
# data from the database about the item)
for item in tomorrow_items:
post_data = model.get_post(int(item.helpRequestId))
contact_data = model.get_contact_data(post_data.contactId)
item.contactName = contact_data.name
item.contactEmail = contact_data.email
item.contactPhone = contact_data.phone
item.url = "%s/view/%s" % (config.SITE_BASE, item.helpRequestId)
item.date = utils.convert_date(item.date)
if not item.helpName:
# If the item doesn't have a name, no one is signed up. Notify the
# the contact person that the spot is still free.
f = web.config.smtp_username
to = item.contactEmail
subject = 'No one signed up for help on %(date)s' % item
msg = """No one signed up for help on %(date)s, item "%(description)s"
Details can be found here:
%(url)s""" % item
elif item.helpEmail:
f = web.config.smtp_username
to = item.helpEmail
MONGO_ARGS = config['MONGO_ARGS']
AUTHOR_BLOCKLIST = config['NLP']['AUTHOR_BLOCKLIST']
NAME_PATTERNS = config['NLP']['NAME_PATTERNS']
MAX_BODY_LENGTH = config['NLP']['MAX_BODY_LENGTH']
DB_NAME = args['db']
READ_COL = args['readcol']
WRITE_COL = args['writecol']
GENDER_IP = args['gr_ip']
GENDER_PORT = args['gr_port']
DOC_LIMIT = args['limit']
force_update = args['force_update']
poolsize = args['poolsize']
chunksize = args['chunksize']
date_begin = utils.convert_date(args['begin_date']) if args['begin_date'] else None
date_end = utils.convert_date(args['end_date']) if args['begin_date'] else None
date_filters = []
if date_begin:
date_filters.append({"publishedAt": {"$gte": date_begin}})
if date_end:
date_filters.append({"publishedAt": {"$lt": date_end + timedelta(days=1)}})
GENDER_RECOGNITION_SERVICE = 'http://{}:{}'.format(GENDER_IP, GENDER_PORT)
if force_update:
other_filters = [
{'quotes': {'$exists': True}}
]
else:
def fieldval_to_id_in_db(self, date): ''' expects a datetime object, returns db id ''' return convert_date(date)
import sys
import re
import bisect
from xml.dom.minidom import parse
import datetime
import serifxml
import serifutils
sys.path.append('/home/hltcoe/bahn/wikitopics/src/wiki')
import utils
if __name__=='__main__':
if len(sys.argv) != 4:
print "pick_self_xml.py DATE TEXT SERIF_XML"
sys.exit(1)
doc = serifxml.Document(sys.argv[3])
date = utils.convert_date(sys.argv[1])
vm2s = dict([(v, s) for s in doc.sentences for v in s.value_mention_set]) # value_mention to sentence
vm2v = dict([(v.value_mention, v) for v in doc.value_set]) # value_mention to value
m2s = dict([(m, s) for s in doc.sentences for m in s.mention_set]) # mention to sentence
m2e = dict([(m, e) for e in doc.entity_set for m in e.mentions]) # mention to entity
first_sentences = [s for s in doc.sentences if s.start_char == 0]
first_entities = set([m2e[m] for s in first_sentences for m in s.mention_set if m in m2e])
candidate_sentences = set([m2s[m] for e in first_entities for m in e.mentions]) - set(first_sentences)
timex_values = set([v for v in doc.value_set if v.value_type == 'TIMEX2.TIME' and v.timex_val and vm2s[v.value_mention] in candidate_sentences])
if not timex_values:
timex_values = set([v for v in doc.value_set if v.value_type == 'TIMEX2.TIME' and v.timex_val]) # fallback
if timex_values:
best_timex = min(timex_values, key=lambda v: serifutils.delta_date_timex(date, v.timex_val))
if timedelta(days=0) < t_delta < timedelta(days=1):
# print 'SEND EMAIL:',item.date,t_delta
tomorrow_items.append(item)
# For each item, if a helper has signed up, send an email reminder (using
# data from the database about the item)
for item in tomorrow_items:
post_data = model.get_post(int(item.helpRequestId))
contact_data = model.get_contact_data(post_data.contactId)
item.contactName = contact_data.name
item.contactEmail = contact_data.email
item.contactPhone = contact_data.phone
item.url = "%s/view/%s" % (config.SITE_BASE, item.helpRequestId)
item.date = utils.convert_date(item.date)
if not item.helpName:
# If the item doesn't have a name, no one is signed up. Notify the
# the contact person that the spot is still free.
f = web.config.smtp_username
to = item.contactEmail
subject = 'No one signed up for help on %(date)s' % item
msg = """No one signed up for help on %(date)s, item "%(description)s"
Details can be found here:
%(url)s""" % item
elif item.helpEmail:
f = web.config.smtp_username
to = item.helpEmail
