def getDay(date):
day = []
es = events.find({"date": date.__str__()})
targetedEvents = []
for e in es:
e["type"] = "event"
targetedEvents.append(e)
targetedEvents = sorted(targetedEvents, key=itemgetter("endTime"))
ds = deadlines.find({"date": date.__str__()})
targetedDeadlines = []
for d in ds:
d["type"] = "deadline"
targetedDeadlines.append(d)
targetedDeadlines = sorted(targetedDeadlines, key=itemgetter("endTime"))
dIdx = eIdx = 0
while dIdx < len(targetedDeadlines) and eIdx < len(targetedEvents):
if targetedDeadlines[dIdx]["endTime"] < targetedEvents[eIdx]["endTime"]:
day.append(targetedDeadlines[dIdx])
dIdx += 1
else:
day.append(targetedEvents[eIdx])
eIdx += 1
day = day + targetedDeadlines[dIdx:] + targetedEvents[eIdx:]
return day
def __init__(self, days):
self.days = days
self.start = date.__str__(date.today())
self.day = date.__str__(date.today() + timedelta(days))
self.end = date.__str__(date.today() + timedelta(days + 4))
self.url = "https://ca.finance.yahoo.com/calendar/earnings?from=" + self.start + "&to=" + self.end + "&day=" + self.day
self.pages = scraper.__general__(url)
self.table = self.table()
self.text = self.text()
def write_aligned_articles_to_file2(articles, valid_ids, out_dir):
languages = ['fi', 'sv']
text_data = {lang: {} for lang in languages}
for lang in languages:
art_count = 0
for art in articles[lang]:
art_id = art['article_id']
#print("Art id:", art_id)
if art_id in valid_ids:
date = art['date']
date_str = date.__str__()
date_str += "-" + str(art_count - (art_count % 20))
print("Date -", lang, ":", date_str)
header = "||Article_id:" + str(art_id) + "||"
article_text_data = art['content']
if date_str in text_data[lang].keys():
text_data[lang][
date_str] += "\n" + header + article_text_data
else:
text_data[lang][date_str] = header + article_text_data
art_count += 1
for lang in languages:
print("Language: ", lang.upper())
dates = text_data[lang].keys()
parent_dir = "../data/yle/" + out_dir + "/"
for dat in dates:
print("Date: ", dat)
fname = parent_dir + dat + "_" + lang + ".txt"
with open(fname, 'w') as fp:
fp.write(text_data[lang][dat])
fp.close()
print("Saved file as: ", fname)
print("Done writing all articles as raw text!")
def write_articles_to_file(path):
fp = open(path, "r")
data = json.load(fp)
languages = list(data.keys())
text_data = {lang: {} for lang in languages}
art_count = len(data['fi'])
for i in range(art_count):
print("Art count: ", i)
date = data['fi'][i]['date']
date = date.__str__()
header = "||Article_id:" + str(i + 1) + "||"
text_data_fi = data['fi'][i]['content']
text_data_sv = data['sv'][i]['content']
if date in text_data['fi'].keys():
text_data['fi'][date] += "\n" + header + text_data_fi
text_data['sv'][date] += "\n" + header + text_data_sv
else:
text_data['fi'][date] = header + text_data_fi
text_data['sv'][date] = header + text_data_sv
#write articles to text files
dates = text_data['fi'].keys()
parent_dir = "data/yle/raw_text/"
for dat in dates:
print("Date: ", dat)
for lang in languages:
fname = parent_dir + dat + "_" + lang + ".txt"
fp = open(fname, 'w')
fp.write(text_data[lang][dat])
fp.close()
print("Saved file as: ", fname)
print("Done writing all articles as raw text!")
def _create_bill(self, date, price, link):
bill = Bill()
bill.id = date.__str__().replace('-', '')
bill.date = date
bill._price = price
bill._url = link
bill.format = u'pdf'
bill.label = unicode(price)
return bill
def perform_smile_skew_hist(symbol):
db=MySQLdb.connect(config.host,config.user,config.password,'NSE')
skew_hist_data=pd.DataFrame(columns=["TIME","SYMBOL","ATM_VOL","SKEW","SMILE","PUT_CALL_RATIO"])
all_dates=unique_date(symbol)
for date in all_dates.timestamp:
data=get_opt_vol_data(symbol, date)
if data.empty:
continue
print symbol,date.__str__()
(atm_vol,skew,smile,put_call_ratio)=calculate_atm_skew_smile(data)
skew_hist_data=skew_hist_data.append({"TIME":date,"SYMBOL":symbol,"ATM_VOL":atm_vol,"SKEW":skew,"SMILE":smile,"PUT_CALL_RATIO":put_call_ratio},ignore_index=1)
skew_hist_data.to_sql("ATM_SKEW_SMILE_HIST",db,flavor='mysql', if_exists='append', chunksize=200)
db.close()
def collect(self, date):
for day in self.day_list:
if date.__str__() == day.date.__str__():
return day
if date <= date.today():
new = self.fetchDay(date)
elif (date - date.today()).days <= 7:
new = self.fetchForecast(date)
else:
raise ValueError("Can't forecast more than 7 days in advance")
self.day_list.append(new)
try:
newfile = open("Weatherdata.pkl", "wb")
pickle.dump(self.day_list, newfile)
newfile.close()
except IOError:
pass
#print "IOEroror"
return new
# Local variables # --------------------------------------------------------------- c = r"V:\Secure\Master_Rail" # working directory d = string.replace(c, "\\", '\\\\') e = d + os.sep + os.sep + "Temp" f = string.replace(c, "\\", '/') + "/Programs" rail_test = e railnet_arc = "railnet_arc" railnet_route_rail_lines = "railnet_route_rail_lines" mrn_gdb = d + os.sep + os.sep + "mrn.gdb" railnet = mrn_gdb + os.sep + os.sep + "railnet" railrt = railnet + "\\railnet_route_rail_lines" Temp = e t = date.today() x = date.__str__(t) x1 = string.replace(x, "-", "") orig_itinerary_dbf = d + os.sep + os.sep + "itin_" + x1 + ".dbf" itinerary = d + os.sep + os.sep + "mrn.gdb" + os.sep + os.sep + "itinerary" new_segments_dbf = e + "\\new_segments.dbf" temp_route_shp = e + os.sep + os.sep + "temp_route.shp" outFl = e + os.sep + os.sep + "geom_out.txt" infl = e + os.sep + os.sep + "geom_in.txt" rte_updt = e + os.sep + os.sep + "rte_updt.dbf" rte_updt_View = "rte_updt_View" test = railnet + os.sep + os.sep + "test" ##set up to run SAS program bat = f + "/sasrun.bat" # batch file name fl = "geometry_update" # SAS file name z = f + "/" + fl + ".sas"
def get_date(self, date): self.INTERFACE.ids.dateLivraison.text = date.__str__()
def collectByDate(self, device, date):
data_list = self.db.collect("mysql315.loopia.se", "kthstud@a68445", "2013IIstud!#", "aktivahuset_com", "meterevents", device, True)
print "Filtering.."
data_list = filter(lambda data: data[:data.find(" ")] in date.__str__(), data_list)
return data_list
from datetime import date
tues = date(2011, 9, 12)
print(repr(tues))
print(str(tues))
print(tues.__repr__())
print(date.__repr__(tues))
print(tues.__str__())
print(date.__str__(tues))
print(len('Go Bears!'))
print('Go Bears!'.__len__())
print(bool(''))
print(bool([]))
print(bool('Go Bears!'))
print('Go Bears!'[3])
print('Go Bears!'.__getitem__(3))
def make_adder(n):
def adder(k):
return n + k
return adder
add_three = make_adder(3)
print(add_three(4))
class Adder:
def __init__(self, n):
self.n = n
