def getRunInfo(self):
logfiles = self.config.get('RunInfo', 'fileName')
logfiles = self.configdir + '/' + logfiles
contentDesc = self.contentDescRunInfo
key = self.config.get('RunInfo', 'key')
runInfo = {}
f = open(logfiles)
for line in f.readlines():
if line != '':
line = line.split()
content = line
thisInfo = {}
unknown = 0
for i in range(len(content)):
if i <= len(contentDesc):
if len(contentDesc[i]) > 2:
thisInfo[contentDesc[i][0]] = utilities.get_value(content[i], contentDesc[i][1],
contentDesc[i][2])
else:
thisInfo[contentDesc[i][0]] = utilities.get_value(content[i], contentDesc[i][1])
else:
thisInfo['unknown%s' % unknown] = content[i]
unkown += 1
if self.verbosity: print thisInfo
if thisInfo.has_key(key):
runInfo[thisInfo[key]] = thisInfo
return runInfo
def get_runlist_map(self):
fileName = self.config.get('RunList', 'fileName')
fileName = self.configdir + '/' + fileName
runList = {}
contentDesc = self.contentDescRunList
key = self.config.get('RunList', 'key')
out = 'Reading RunListMap with the following keys '+str(key)+' and '+str(contentDesc)
raw_input(out)
with open(fileName, 'rb') as csvfile:
reader = csv.reader(csvfile, delimiter=',')
l = 0
for row in reader:
thisRun = {}
for i in range(len(row)):
if len(contentDesc[i]) > 2:
thisRun[contentDesc[i][0]] = utilities.get_value(row[i], contentDesc[i][1], contentDesc[i][2])
else:
print row[i], type(row[i])
thisRun[contentDesc[i][0]] = utilities.get_value(row[i], contentDesc[i][1])
if thisRun.has_key(key):
runList[thisRun[key]] = thisRun
else:
print 'cannot find key, ', key, 'in thisRun', thisRun
if l == 0:
print thisRun
raw_input('Row %s'%row)
l+=1
return runList
def get_repeaterCard_map(self):
fileName = self.config.get('RepeaterCards', 'fileName')
fileName = self.configdir + '/' + fileName
contentDesc = self.contentDescRepCard
if self.verbosity: print contentDesc
key = self.config.get('RepeaterCards', 'key')
repeaterCard_and_current_File = open(fileName)
repeaterCardMap = {}
i = 0
for line in repeaterCard_and_current_File.readlines():
i += 1
if line.startswith('#'):
continue
line = line.replace('\n', '').replace('\r', '')
content = line.split(',')
thisRepeaterCard = {}
if i == 1:
continue
unknown = 0
for i in range(len(content)):
if i <= len(contentDesc):
if len(contentDesc[i]) > 2:
thisRepeaterCard[contentDesc[i][0]] = utilities.get_value(content[i], contentDesc[i][1],
contentDesc[i][2])
else:
thisRepeaterCard[contentDesc[i][0]] = utilities.get_value(content[i], contentDesc[i][1])
else:
thisRepeaterCard['unknown%s' % unknown] = content[i]
unkown += 1
if self.verbosity: print thisRepeaterCard
if thisRepeaterCard.has_key(key):
repeaterCardMap[thisRepeaterCard[key]] = thisRepeaterCard
return repeaterCardMap
def is_corrected(config):
# config.get('RunInfo','cor')
input1 = config.get("RunInfo", "realRunNo")
input2 = config.get("RunInfo", "runno")
runNo = utilities.get_value(input2, "int", "-1")
realRunNo = utilities.get_value(input1, "int", "-1")
retVal = 0
if realRunNo == runNo:
retVal = 0
else:
retVal = 1
if int(runNo / 1e5) > 0:
retVal = 1
# else:
# retVal = 1
# print 'is_corrected:'
# print ' * Input1',input1
# print ' * Input1',input2
# print ' * realRunNo',realRunNo
# print ' * runNo',runNo
# print ' * 1e5:',(int(runNo / 1e5) > 0),int(runNo / 1e5)
# print ' * realrunno:',config.get('RunInfo', 'realrunno')
# print ' => ',retVal
# raw_input()
return retVal
def get_runlist_map(self):
fileName = self.config.get("RunList", "fileName")
fileName = self.configdir + "/" + fileName
runList = {}
contentDesc = self.contentDescRunList
key = self.config.get("RunList", "key")
out = " - Reading RunListMap with the following keys " + str(key) + " and " + str(contentDesc)
# raw_input(out)
with open(fileName, "rb") as csvfile:
reader = csv.reader(csvfile, delimiter=",")
l = 0
for row in reader:
thisRun = {}
for i in range(len(row)):
if len(contentDesc[i]) > 2:
thisRun[contentDesc[i][0]] = utilities.get_value(row[i], contentDesc[i][1], contentDesc[i][2])
else:
print " - ", row[i], type(row[i])
thisRun[contentDesc[i][0]] = utilities.get_value(row[i], contentDesc[i][1])
if thisRun.has_key(key):
runList[thisRun[key]] = thisRun
else:
print "cannot find key, ", key, "in thisRun", thisRun
if l == 0:
print " - ", thisRun
# raw_input('Row %s'%row)
l += 1
nMissing = []
for run in self.all_runlogs:
if run["runNo"] not in runList:
print "missing run", run["runNo"]
nMissing.append(run["runNo"])
thisInfo = {}
for i in range(0, len(contentDesc)):
try:
content = str(run[contentDesc[i][0]])
except Exception as e:
content = "nan"
if len(contentDesc[i]) > 2:
try:
thisInfo[contentDesc[i][0]] = utilities.get_value(
content, contentDesc[i][1], contentDesc[i][2]
)
except Exception as e:
print "cannot convert", content, contentDesc[i]
raise e
else:
thisInfo[contentDesc[i][0]] = utilities.get_value(content, contentDesc[i][1])
runList[run["runNo"]] = thisRun
print "Added ", nMissing, " runs to runList which were missing, total runs read from test beam logs:", len(
self.all_runlogs
)
return runList
def getRunInfo(self):
logfiles = self.config.get("RunInfo", "fileName")
logfiles = self.configdir + "/" + logfiles
contentDesc = self.contentDescRunInfo
key = self.config.get("RunInfo", "key")
runInfo = {}
f = open(logfiles)
for line in f.readlines():
if line != "":
line = line.split()
content = line
thisInfo = {}
unknown = 0
for i in range(len(content)):
if i <= len(contentDesc):
if len(contentDesc[i]) > 2:
thisInfo[contentDesc[i][0]] = utilities.get_value(
content[i], contentDesc[i][1], contentDesc[i][2]
)
else:
thisInfo[contentDesc[i][0]] = utilities.get_value(content[i], contentDesc[i][1])
else:
thisInfo["unknown%s" % unknown] = content[i]
unknown += 1
if self.verbosity:
print thisInfo
if thisInfo.has_key(key):
runInfo[thisInfo[key]] = thisInfo
print "Unknown items", unknown
nMissing = 0
for run in self.all_runlogs:
if run["runNo"] not in runInfo:
print "missing run", run["runNo"]
nMissing += 1
thisInfo = {}
for i in range(0, len(contentDesc)):
content = str(run[contentDesc[i][0]])
if len(contentDesc[i]) > 2:
try:
thisInfo[contentDesc[i][0]] = utilities.get_value(
content, contentDesc[i][1], contentDesc[i][2]
)
except Exception as e:
print "cannot convert", content, contentDesc[i]
raise e
else:
thisInfo[contentDesc[i][0]] = utilities.get_value(content, contentDesc[i][1])
runInfo[thisInfo[key]] = thisInfo
print "Added ", nMissing, " runs which were missing, total runs read from test beam logs:", len(
self.all_runlogs
)
return runInfo
def is_corrected(config):
input1 = config.get('RunInfo', 'realRunNo')
input2 = config.get('RunInfo', 'runno')
runNo = utilities.get_value(input2, 'int', '-1')
realRunNo = utilities.get_value(input1, 'int', '-1')
retVal = 0
if realRunNo == runNo:
retVal = 0
else:
retVal = 1
if int(runNo / 1e5) == 0:
retVal = 0
else:
retVal = 1
return retVal
def getRunInfo(self):
logfiles = self.config.get('RunInfo', 'fileName')
logfiles = self.configdir + '/' + logfiles
contentDesc = self.contentDescRunInfo
key = self.config.get('RunInfo', 'key')
runInfo = {}
f = open(logfiles)
for line in f.readlines():
if line != '':
line = line.split()
content = line
thisInfo = {}
unknown = 0
for i in range(len(content)):
if i <= len(contentDesc):
if len(contentDesc[i]) > 2:
thisInfo[contentDesc[i][0]] = utilities.get_value(content[i], contentDesc[i][1],
contentDesc[i][2])
else:
thisInfo[contentDesc[i][0]] = utilities.get_value(content[i], contentDesc[i][1])
else:
thisInfo['unknown%s' % unknown] = content[i]
unknown += 1
if self.verbosity: print thisInfo
if thisInfo.has_key(key):
runInfo[thisInfo[key]] = thisInfo
print 'Unknown items',unknown
nMissing =0
for run in self.all_runlogs:
if run['runNo'] not in runInfo:
print 'missing run',run['runNo']
nMissing += 1
thisInfo = {}
for i in range(0,len(contentDesc)):
content = str(run[contentDesc[i][0]])
if len(contentDesc[i]) > 2:
try:
thisInfo[contentDesc[i][0]] = utilities.get_value(content, contentDesc[i][1],
contentDesc[i][2])
except Exception as e:
print 'cannot convert',content,contentDesc[i]
raise e
else:
thisInfo[contentDesc[i][0]] = utilities.get_value(content, contentDesc[i][1])
runInfo[thisInfo[key]] = thisInfo
print 'Missing',nMissing,'of',len(self.all_runlogs)
raw_input()
return runInfo
def get_content(self, result):
row = []
for key in self.header_mapping:
# print key
mainLink = resultsCreation.get_mainLink(result, self.config.get('HTML', 'absolutePath'))
result.set('RunInfo', 'mainLink', mainLink)
if self.header_mapping[key].has_key('key'):
default = self.header_mapping[key]['default']
configKeys = self.header_mapping[key]['key']
if result.has_section(configKeys[0]):
if result.has_option(configKeys[0], configKeys[1]):
value = result.get(configKeys[0], configKeys[1], 0)
else:
value = default
else:
value = default
if type(value) == list and len(value) == 1:
value = value[0]
value_type = self.header_mapping[key]['valueType']
value = utilities.get_value(value, value_type, default)
# print key,configKeys,value
elif self.header_mapping[key].has_key('title'):
value = self.header_mapping[key]['title']
else:
value = 'UNKOWN'
pass
row.append(self.get_cell(value, key, result))
return row
def get_repeaterCard_map(self):
fileName = self.config.get("RepeaterCards", "fileName")
fileName = self.configdir + "/" + fileName
contentDesc = self.contentDescRepCard
if self.verbosity:
print contentDesc
key = self.config.get("RepeaterCards", "key")
repeaterCard_and_current_File = open(fileName)
repeaterCardMap = {}
i = 0
for line in repeaterCard_and_current_File.readlines():
i += 1
if line.startswith("#"):
continue
line = line.replace("\n", "").replace("\r", "")
content = line.split(",")
thisRepeaterCard = {}
if i == 1:
continue
unknown = 0
for i in range(len(content)):
if i <= len(contentDesc):
if len(contentDesc[i]) > 2:
thisRepeaterCard[contentDesc[i][0]] = utilities.get_value(
content[i], contentDesc[i][1], contentDesc[i][2]
)
else:
thisRepeaterCard[contentDesc[i][0]] = utilities.get_value(content[i], contentDesc[i][1])
else:
thisRepeaterCard["unknown%s" % unknown] = content[i]
unkown += 1
if self.verbosity:
print thisRepeaterCard
if thisRepeaterCard.has_key(key):
repeaterCardMap[thisRepeaterCard[key]] = thisRepeaterCard
return repeaterCardMap
def __init__(self, api_timestamp, xml):
self.api_timestamp = api_timestamp
self.product_id = utilities.get_value(int, xml, 'ProductID[@type="Reference"]', False)
self.title = utilities.get_value(str, xml, 'Title', False)
self.review_count = utilities.get_value(int, xml, 'ReviewCount', True)
self.domain_name = utilities.get_value(str, xml, 'DomainName', True)
self.details_url = utilities.get_value(str, xml, 'DetailsURL', True)
self.stock_photo_url = utilities.get_value(str, xml, 'StockPhotoURL', True)
def __init__(self, api_timestamp, xml):
self.api_timestamp = api_timestamp
self.best_offer_enabled = utilities.get_value(bool, xml,
'BestOfferEnabled', True)
self.auto_pay_enabled = utilities.get_value(bool, xml,
'AutoPayEnabled', True)
self.category_id = utilities.get_value(int, xml, 'CategoryID', False)
self.category_level = utilities.get_value(int, xml, 'CategoryLevel',
False)
self.category_name = utilities.get_value(str, xml, 'CategoryName',
False)
self.category_parent_id = utilities.get_value(int, xml,
'CategoryParentID',
False)
self.category_enabled = False
def add_default(self, item, contentDesc):
for i in contentDesc:
item[i[0]] = utilities.get_value(i[2], i[1], i[2])
pass
return item
def get_csv_row(self, config):
row = []
# result = self.results
for key in self.csv_mapping:
if 'event' in key:
pass
#print 'get csv row',key,self.csv_mapping.keys(),self.csv_mapping[key]
result = config
if 'key' in self.csv_mapping[key]:
if 'event' in key:
print 'get csv row',self.csv_mapping[key]
try:
default = self.csv_mapping[key]['default']
except:
default = 'unknown'
configKeys = self.csv_mapping[key]['key']
if result.has_section(configKeys[0]):
if result.has_option(configKeys[0], configKeys[1]):
value = result.get(configKeys[0], configKeys[1], 0)
else:
if 'event' in key:
print 'need to take default'
value = default
else:
if 'event' in key:
print 'need to take default'
value = default
try:
value_type = self.csv_mapping[key]['valueType']
except:
value_type = 'string'
value = utilities.get_value(value, value_type, default)
if 'event' in key:
print key,configKeys,value
else:
if 'event' in key:
# print 'cannot find mapping',key
pass
try:
if 'title' in self.header_mapping[key]:
value = self.csv_mapping[key]['title']
except:
value = 'UNKOWN'
pass
try:
string_format = self.config.get('FORMAT', key, '%s')
except:
print 'cannot find format for key', key
string_format = '%s'
if 'event' in key:
print key,string_format, value,
try:
out = string_format % value
except:
print 'cannot format "%s"/"%s" for key %s"' % (string_format, value, key),default
out = None
if out is None:
try:
out = value
except:
out = 'UNKNOWN'
out = out.strip('"')
row.append(out)
return row
pass
def __init__(self, api_timestamp, xml):
self.api_timestamp = api_timestamp
self.item_id = get_value(int, xml, 'itemId', False)
self.title = get_value(str, xml, 'title', False)
self.subtitle = get_value(str, xml, 'subtitle', True)
self.global_id = get_value(str, xml, 'globalId', False)
self.category_id = get_value(int, xml, 'primaryCategory/categoryId', False)
self.category_name = get_value(str, xml, 'primaryCategory/categoryName', False)
self.gallery_url = get_value(str, xml, 'galleryURL', True)
self.view_item_url = get_value(str, xml, 'viewItemURL', True)
self.product_id = get_value(int, xml, 'productId[@type="ReferenceID"]', False)
# Payment Info
self.pi_payment_method = get_values(xml, 'paymentMethod', True)
self.pi_auto_pay = get_value(bool, xml, 'autoPay', True)
self.pi_postal_code = get_value(str, xml, 'postalCode', True)
self.pi_location = get_value(str, xml, 'location', True)
self.pi_country = get_value(str, xml, 'country', True)
# Shipping Info
self.si_shipping_service_cost = get_value(float, xml, 'shippingInfo/shippingServiceCost', True)
self.si_shipping_service_cost_currency_id = get_attribute_value(str, xml, 'shippingInfo/shippingServiceCost','currencyId', True)
self.si_shipping_type = get_value(str, xml, 'shippingInfo/shippingType', True)
self.si_ship_to_locations = get_values(xml, 'shippingInfo/shipToLocations', True)
self.si_expedited_shipping = get_value(bool, xml, 'shippingInfo/expeditedShipping', True)
self.si_one_day_shipping_available = get_value(bool, xml, 'shippingInfo/oneDayShippingAvailable', True)
self.si_handling_time = get_value(int, xml, 'shippingInfo/handlingTime', True)
# Selling Status
self.ss_current_price = get_value(float, xml, 'sellingStatus/currentPrice', True)
self.ss_current_price_currency_id = get_attribute_value(str, xml, 'sellingStatus/currentPrice', 'currencyId', True)
self.ss_converted_current_price = get_value(float, xml, 'sellingStatus/convertedCurrentPrice', True)
self.ss_converted_current_price_currency_id = get_attribute_value(str, xml, 'sellingStatus/convertedCurrentPrice', 'currencyId', True)
self.ss_bid_count = get_value(int, xml, 'sellingStatus/bidCount', True)
self.ss_selling_state = get_value(str, xml, 'sellingStatus/sellingState', True)
self.ss_time_left = get_value(str, xml, 'sellingStatus/timeLeft', True)
# Listing Info
self.li_best_offer_enabled = get_value(bool, xml, 'listingInfo/bestOfferEnabled', True)
self.li_buy_it_now_available = get_value(bool, xml, 'listingInfo/buyItNowAvailable', True)
self.li_buy_it_now_price = get_value(float, xml, 'listingInfo/buyItNowPrice', True)
self.li_buy_it_now_price_currency_id = get_attribute_value(str, xml, 'listingInfo/buyItNowPrice', 'currencyId', True)
self.li_converted_buy_it_now_price = get_value(float, xml, 'listingInfo/convertedBuyItNowPrice', True)
self.li_converted_buy_it_now_price_currency_id = get_attribute_value(str, xml, 'listingInfo/convertedBuyItNowPrice', 'currencyId', True)
self.li_start_time = get_value(str, xml, 'listingInfo/startTime', True)
self.li_end_time = get_value(str, xml, 'listingInfo/endTime', True)
self.li_listing_type = get_value(str, xml, 'listingInfo/listingType', True)
self.li_gift = get_value(str, xml, 'listingInfo/gift', True)
# Other stuff
self.returns_accepted = get_value(str, xml, 'returnsAccepted', True)
self.condition_id = get_value(int, xml, 'condition/conditionId', True)
self.condition_display_name = get_value(str, xml, 'condition/conditionDisplayName', True)
self.is_multi_variation_listing = get_value(bool, xml, 'isMultiVariationListing', True)
self.top_rated_listing = get_value(bool, xml, 'topRatedListing', True)
