def run(cls, file, *args, **kwargs):
"""
Run this command and return a generator of filtered output lines
Note: This is not profilable so subclasses should override it to make
sure it's taken into account by --profile
"""
regex = cls.make_regex(file)
cmd = cls.make_cmd(file, *args, **kwargs)
# Handle empty commands to save some time
if cmd is None:
return iter(())
Logger.debug("Running command {}".format(" ".join(cmd)))
process = subprocess.run(cmd,
shell=False,
close_fds=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if process.stderr:
text = "Error while running command \"{}\": {}".format(
" ".join(cmd), process.stderr.decode("utf-8"))
Logger.warn(text)
return cls._filter(process.stdout.splitlines(True), regex)
def is_increasing_consecutively(symbol, iexHistoricalData, repeats):
Logger.debug('analyzing {} for {} repeated increasing minutes'.format(
symbol, repeats))
chainLength = 0
failedChains = 0
beLessThanMeChainHead = None
for i in range(len(iexHistoricalData) - 1, -1, -1):
minuteData = iexHistoricalData[i]
currentMinuteAverage = minuteData.get('average')
#many trading minutes have no activity
if (currentMinuteAverage != None):
if (beLessThanMeChainHead == None):
beLessThanMeChainHead = currentMinuteAverage
elif (currentMinuteAverage < beLessThanMeChainHead):
Logger.debug('increasing consecutive count')
chainLength += 1
elif (currentMinuteAverage >= beLessThanMeChainHead):
chainLength = 0
failedChains += 1
if (chainLength >= repeats):
Logger.debug(
'successful increasing successfully check on {} for {} repeats'
.format(symbol, repeats))
return True
if (failedChains >= 3):
Logger.debug(
'aborting consecutive analysis of {}'.format(symbol))
return False
beLessThanMeChainHead = currentMinuteAverage
return False
def hook_after_request(response):
route = request.path
logging_method = Logger.error if response.status_code not in info_status_list else Logger.info
Logger.debug(request)
try:
req_body = request.get_json()
except Exception:
req_body = 'No JSON in request'
res_body = response.get_data().decode('utf-8').rstrip()
logging_method(
message='\nRequest: {}\nResponse: {}'.format(req_body, res_body),
route=route,
method=request.method,
res_code=response.status_code
)
return response
def valueOverRangeLineGraph(ticker, value, start, end):
dailyData = iex.getHistoricalDataByRange(ticker, start, end)
print('here1, should log iex historvatl range')
Logger.debug('iex historical range data is')
Logger.debug(dailyData)
seriesData = {}
for day, value in dailyData.items():
print('day is')
print(day)
print(value)
seriesData.update({pd.to_datetime(day): value.get('close')})
openingByDaySeries = pd.Series(seriesData)
graph = openingByDaySeries.plot.line()
graph.set_xlabel("Day")
graph.set_ylabel(value)
Logger.writeGraphToFile(
graph.get_figure(),
"_".join([ticker, str(start),
str(end), "Closing Price"]))
def quick_trade_one(totalToSpend, stockQueue):
Logger.debug('Calling quick_trade_one with totalToSpend : {}'.format(
totalToSpend))
spent = 0
investedStocks = 0
# while (spent < totalToSpend):
while (spent < totalToSpend):
stock = stockQueue.get().get('symbol')
byMinuteData = iex.getHistoricalIntradayByMinute(stock)
if (BetaTrades.is_increasing_consecutively(stock, byMinuteData,
8)):
#make bracket order
askprice = Alpaca.getAskPrice(stock)
take_price = askprice * 1.02
stop_price = askprice * .988
quantity = math.floor(totalToSpend / askprice)
Alpaca.buy_market_bracket_order(stock, take_price, stop_price,
quantity)
Logger.all(
'quick_trade_one triggering bracket trade on {} at ask price {}, take price {}, stop price {}'
.format(stock, askprice, take_price, stop_price))
investedStocks += 1
class Hemnet():
def __init__(self):
self.log = Logger("Hemnet")
self.request = Request()
#Base objects for searches and results
self.baseUrl = "http://www.hemnet.se"
self.baseSearch = self.baseUrl + "/sok/create"
self.baseLocation = self.baseUrl + "/locations/show?"
self.baseResult = self.baseUrl + "/resultat"
self.searchQuery = {}
#Basetype, english -> Swedish
self.translatedTypes = {
"municipality": "Kommun",
"district": u"Område",
"postal_city": "Stadsdel",
"region": u"Län",
"street": "Gata",
"city": "Stad"
}
#BaseAverageTypes -> Swedish
self.translatedAverageTypes = {
"age": u"List ålder",
"price": "Medelpris",
"price_m2": u"Pris per m²",
"size": u"Storlek (m²)",
"rooms": "Antal rum",
"fee": u"Månadsavgift",
"price_change_up": u"Prisökning (%)",
"price_change_down": u"Prissäkning (%)"
}
#searchTypes
self.searchTypes = {
"f": "fritidshus",
"v": "villa",
"t": "tomt",
"r": "radhus",
"g": "gard",
"b": "bostadsratt",
"o": "other",
"a": "all"
}
#Items to get average for
self.itemAverageTypes = {
"age": 0,
"price": 0,
"price_m2": 0,
"size": 0,
"rooms": 0,
"fee": 0,
"price_change_up": 0,
"price_change_down": 0
}
#Base result format
self.resultFormat = {
"totalItems": 0,
"results": {}
}
self.log.info("Initiated Hemnet")
'''
Searchdata is a formpost in a very specific format
'''
def createSearchFormData(self, data, specificType='a'):
locationData = [{
"id": (data.get("id")),
"name": (data.get("name")),
"parent_id": (data.get("parent_location").get("id")),
"parent_name": (data.get("parent_location").get("name"))
}]
searchData = {
"search[location_search]": locationData,
"search[location_ids][]": data.get("id"),
"search[region_id]": -1,
"search[municipality_ids][]": -1,
"search[country_id]": 0,
"search[item_types][]": "%s" % self.searchTypes[specificType],
"search[price_min]": '',
"search[price_max]": '',
"search[fee_max]": '',
"search[rooms_min]": '',
"search[living_area_min]": '',
"search[keywords]": '',
"commit": ''
}
return searchData
def searchRequest(self, query):
return self.request.postRequest(self.baseSearch, query)
'''
Pass a list of keys and a dict of data to caluclate average value for each key
'''
def avgByKey(self, keys, data):
final = {}
for d in data:
for k in d.keys():
if k in keys:
final[k] = final.get(k, 0) + d[k]
for k in final.keys():
final[k] = final[k] / len(data)
return final
def getLocationQueryURL(self, query):
return "%sq=%s" % (self.baseLocation,
urllib.quote(query.encode('utf-8')))
@cache.methodcache.cache('findLocations', expire=72000)
def findLocations(self, query, extra, area=None):
queryURL = self.getLocationQueryURL(query)
cacheResult = cache.locations.get(hashlib.md5(queryURL).hexdigest())
if (cacheResult is not None):
print "Found cached loc"
return cacheResult
locFormData = []
locResponse = self.request.getResponse(queryURL, None)
jdata = json.loads(locResponse)
print json.dumps(jdata, indent=4)
formData = {}
locations = []
for id, item in enumerate(jdata):
item["score"] = Levenshtein.ratio(
item.get("location").get("name"), query)
if (area is not None):
if (item.get("location").get("parent_location").get(
"name").find(area) != -1):
formData = self.createSearchFormData(
item.get("location"), extra)
locations.append(item)
locFormData.append(formData)
else:
formData = self.createSearchFormData(item.get("location"),
extra)
locations.append(item)
locFormData.append(formData)
locations = sorted(locations, key=itemgetter('score'), reverse=True)
result = {
'search': locFormData,
'area': area,
'locations': locations
}
cache.locations[hashlib.md5(queryURL).hexdigest()] = result
return result
@cache.methodcache.cache('performSearch', expire=72000)
def performSearch(self, searchData):
hashkey = hashlib.md5(json.dumps(searchData,
sort_keys=True)).hexdigest()
cachedResult = cache.storage.get(hashkey)
if (cachedResult is not None):
print "Found cached searchResponse"
return cachedResult
print "Performing search on " + json.dumps(searchData, indent=4)
searchRequest = self.searchRequest(searchData)
searchResponse = self.request.getUnicodeDoc(searchRequest)
resultData = self.parseResult(searchResponse, self.resultFormat)
result = self.createResultItem(resultData)
print "Storing hash " + hashkey
chart_list = cache.storage.get(hashkey, {})
# metadata is the chart item minus the actual list plus a size
metadata_keys = filter(lambda k: k != 'list', result.keys())
metadata = {key: result[key] for key in metadata_keys}
chart_list[hashkey] = metadata
cache.storage[hashkey] = chart_list[hashkey]
return result
def parseResult(self, doc, brokers={}):
brokers = self.parseItems(
doc.xpath("//div[contains(@class, 'item result')]"), brokers)
nextpage = doc.xpath('//a[@class="next_page"]')
try:
url = nextpage[0].attrib["href"]
if url is not None:
self.log.info("Parsing %s" % url)
nextDoc = self.request.requestUnicodeDoc(self.baseUrl + url)
self.parseResult(nextDoc, brokers)
except Exception, e:
self.log.debug("ParseResult %s" % e)
pass
return brokers
class Hemnet() :
def __init__(self):
self.log = Logger("Hemnet");
self.request = Request();
#Base objects for searches and results
self.baseUrl = "http://www.hemnet.se";
self.baseSearch = self.baseUrl + "/sok/create";
self.baseLocation = self.baseUrl + "/locations/show?";
self.baseResult = self.baseUrl + "/resultat";
self.searchQuery = {}
#Basetype, english -> Swedish
self.translatedTypes = {
"municipality" : "Kommun",
"district" : u"Område",
"postal_city" : "Stadsdel",
"region" : u"Län",
"street" : "Gata",
"city" : "Stad"
}
#BaseAverageTypes -> Swedish
self.translatedAverageTypes = {
"age" : u"List ålder",
"price" : "Medelpris",
"price_m2" : u"Pris per m²",
"size" : u"Storlek (m²)",
"rooms" : "Antal rum",
"fee" : u"Månadsavgift",
"price_change_up" : u"Prisökning (%)",
"price_change_down" : u"Prissäkning (%)"
}
#searchTypes
self.searchTypes = {
"f" : "fritidshus",
"v" : "villa",
"t" : "tomt",
"r" : "radhus",
"g" : "gard",
"b" : "bostadsratt",
"o" : "other",
"a" : "all"
}
#Items to get average for
self.itemAverageTypes = {
"age" : 0,
"price" : 0,
"price_m2" : 0,
"size" : 0,
"rooms" : 0,
"fee" : 0,
"price_change_up" : 0,
"price_change_down" : 0
};
#Base result format
self.resultFormat = {
"totalItems" : 0,
"results" : {}
};
self.log.info("Initiated Hemnet");
'''
Searchdata is a formpost in a very specific format
'''
def createSearchFormData(self, data, specificType = 'a') :
locationData = [{
"id": (data.get("id")),
"name": (data.get("name")),
"parent_id": (data.get("parent_location").get("id")),
"parent_name": (data.get("parent_location").get("name"))
}]
searchData = {
"search[location_search]" : locationData,
"search[location_ids][]": data.get("id"),
"search[region_id]":-1,
"search[municipality_ids][]":-1,
"search[country_id]":0,
"search[item_types][]": "%s" % self.searchTypes[specificType],
"search[price_min]": '',
"search[price_max]": '',
"search[fee_max]": '',
"search[rooms_min]": '',
"search[living_area_min]": '',
"search[keywords]":'',
"commit": ''
}
return searchData;
def searchRequest(self, query) :
return self.request.postRequest(self.baseSearch, query);
'''
Pass a list of keys and a dict of data to caluclate average value for each key
'''
def avgByKey(self, keys, data):
final = {}
for d in data:
for k in d.keys():
if k in keys:
final[k] = final.get(k,0) + d[k]
for k in final.keys():
final[k] = final[k]/len(data);
return final;
def getLocationQueryURL(self, query):
return "%sq=%s" % (self.baseLocation, urllib.quote(query.encode('utf-8')))
@cache.methodcache.cache('findLocations', expire=72000)
def findLocations(self, query, extra, area = None) :
queryURL = self.getLocationQueryURL(query);
cacheResult = cache.locations.get(hashlib.md5(queryURL).hexdigest());
if( cacheResult is not None):
print "Found cached loc";
return cacheResult;
locFormData = []
locResponse = self.request.getResponse(queryURL, None)
jdata = json.loads(locResponse);
print json.dumps(jdata, indent=4);
formData = {}
locations = []
for id, item in enumerate(jdata) :
item["score"] = Levenshtein.ratio(item.get("location").get("name"), query)
if( area is not None ):
if( item.get("location").get("parent_location").get("name").find(area) != -1 ):
formData = self.createSearchFormData(item.get("location"), extra);
locations.append(item)
locFormData.append(formData);
else:
formData = self.createSearchFormData(item.get("location"), extra);
locations.append(item)
locFormData.append(formData);
locations = sorted(locations, key=itemgetter('score'), reverse=True)
result = {'search' : locFormData, 'area' : area, 'locations' : locations };
cache.locations[hashlib.md5(queryURL).hexdigest()] = result
return result;
@cache.methodcache.cache('performSearch', expire=72000)
def performSearch(self, searchData):
hashkey = hashlib.md5(
json.dumps(searchData, sort_keys=True)
).hexdigest();
cachedResult = cache.storage.get(hashkey);
if(cachedResult is not None):
print "Found cached searchResponse";
return cachedResult;
print "Performing search on " + json.dumps(searchData, indent=4);
searchRequest = self.searchRequest(searchData);
searchResponse = self.request.getUnicodeDoc(searchRequest);
resultData = self.parseResult(searchResponse, self.resultFormat);
result = self.createResultItem(resultData);
print "Storing hash " + hashkey;
chart_list = cache.storage.get(hashkey, {})
# metadata is the chart item minus the actual list plus a size
metadata_keys = filter(lambda k: k != 'list', result.keys())
metadata = { key: result[key] for key in metadata_keys }
chart_list[hashkey] = metadata
cache.storage[hashkey] = chart_list[hashkey]
return result;
def parseResult(self, doc, brokers = {}) :
brokers = self.parseItems(doc.xpath("//div[contains(@class, 'item result')]"), brokers);
nextpage = doc.xpath('//a[@class="next_page"]');
try:
url = nextpage[0].attrib["href"];
if url is not None:
self.log.info("Parsing %s" % url);
nextDoc = self.request.requestUnicodeDoc(self.baseUrl + url);
self.parseResult(nextDoc, brokers);
except Exception,e:
self.log.debug("ParseResult %s" % e)
pass;
return brokers;
