def loadDataFeed(feed):
logger.info("Loading feed: " + IPUtils.dictToString(feed))
filePath = IPUtils.getDataFeedFilePath(feed)
fileName = filePath + '/' + feed['fileName']
tempFile = fileName + '.tmp'
# Create storage directory if it doesn't already exist
if not os.path.exists(filePath):
os.makedirs(filePath)
# Download the XML file from the URL and save locally
# We first save to a temp file then rename, as the rename is atomic and prevents issues with
# Other processes attempting to read the file while downloading
with urllib.request.urlopen(feed['url']) as response, open(
tempFile, 'wb') as outFile:
data = response.read() # a `bytes` object
outFile.write(data)
os.rename(tempFile, fileName)
logger.info("Loaded data to: " + fileName)
def processPlayer(player):
externalPlayerID = int(player.attrib['id'])
internalPlayerID = playerDataSourceMappings.get(externalPlayerID)
# Only process players for which we have ID mappings after loading team data
if (internalPlayerID != None):
playerName = metadata.XPath.Player_Name(player)[0].text
logger.info("Processing player external_id=" + str(externalPlayerID) +
", internal_id=" + str(internalPlayerID) + ", name=" +
playerName)
# Save player image
if (len(metadata.XPath.Player_Image(player)) > 0):
fileName = None
fileType = None
if (args.storeToS3):
logger.info("Saving player image to Amazon S3")
fileName = 'images/players/' + str(internalPlayerID) + '.jpg'
fileType = IPUtils.saveBase64EncodedImageToAmazonS3(
metadata.XPath.Player_Image(player)[0].text, fileName,
s3bucket)
else:
logger.info("Saving player image to file")
fileName = '/var/tmp/inplayrs/files/images/players/' + str(
internalPlayerID) + '.jpg'
fileType = IPUtils.saveBase64EncodedImageToFile(
metadata.XPath.Player_Image(player)[0].text, fileName)
if (fileType == None):
logger.warning("Invalid image found in tag, unable to save")
else:
logger.info("Successfully saved image. fileType=" + fileType +
", fileName=" + fileName)
else:
logger.info("No player image found")
parser.add_argument("env",
choices=['local', 'dev', 'prod'],
metavar="env",
help="Environment (local/dev/prod)")
parser.add_argument("input_file", help="File containing team data")
parser.add_argument(
"-s3",
"--storeToS3",
help="Store images to Amazon S3, even if environment is not prod",
action="store_true")
parser.add_argument("-d", "--debug", help="Debug mode", action="store_true")
args = parser.parse_args()
# Get logger and set level
loggingLevel = (logging.DEBUG if args.debug else logging.INFO)
logger = IPUtils.getLogger(scriptName, loggingLevel)
# load DB config and connect to DB
dbConfig = config.Connections.dbConfig.get(args.env)
db = pymysql.connect(host=dbConfig['host'],
port=dbConfig['port'],
user=dbConfig['user'],
passwd=dbConfig['pass'],
db=dbConfig['db'])
db.autocommit(1)
# Create Amazon S3 connection (used for storing images)
s3conn = S3Connection(config.Connections.AWS_ACCESS_KEY_ID,
config.Connections.AWS_SECRET_ACCESS_KEY)
s3bucket = s3conn.get_bucket(
'storage.inplayrs.com',
def processTeam(team):
teamIsInCompetiton = False
# Only process the team if they are in the competition (external_comp_id)
for league_id in metadata.XPath.Team_LeagueIDs(team):
if (league_id.text == str(args.external_comp_id)):
teamIsInCompetiton = True
if (not teamIsInCompetiton):
return
teamName = metadata.XPath.Team_Name(team)[0].text
externalTeamID = team.attrib['id']
logger.info("Processing team: "+teamName+", external_id: "+externalTeamID)
# Check if a team with that name exists for that competition (internal_comp_id) in team table, and if not, insert it and keep the ID
if (teams.get(teamName) == None):
# Check if there is an existing mapping first, and if so, throw an error and stop processing this team
if(teamDataSourceMappings.get(externalTeamID) != None):
logger.error("Team with name "+teamName+" does not exist however there is already a data_source_mapping for this team: mapping_id="+
teamDataSourceMappings[externalTeamID]['mapping_id']+", skipping processing of this team. Please update team name in DB if name has changed.")
return
# Insert new team
logger.info("Inserting new team. name="+teamName+", competition="+str(args.internal_comp_id))
newTeamId = insertTeam(teamName, args.internal_comp_id)
if (newTeamId > 0):
logger.info("New team_id for "+teamName+": "+str(newTeamId))
teams[teamName] = newTeamId
# Create new data source mapping for the team
logger.info("Creating new data_source_mapping entry for team: internal_id="+newTeamId+", external_id="+externalTeamID)
newDataSourceMappingID = insertDataSourceMapping(args.data_source_id, 'team', newTeamId, int(externalTeamID))
if (newDataSourceMappingID > 0):
teamDataSourceMappings[externalTeamID] = {"internal_id" : newTeamId, "mapping_id:" : newDataSourceMappingID}
else:
logger.error("Failed to create new data_source_mapping for team with internal_id="+newTeamId+", external_id="+externalTeamID+". skipping processing of this team")
return
else:
logger.error("Failed to insert team "+teamName+", skipping processing of this team")
return
else:
logger.info("Team already exists in DB with team_id "+str(teams.get(teamName)))
# Save team image
if (len(metadata.XPath.Team_Image(team)) > 0 ):
fileName = None
fileType = None
if (args.storeToS3):
logger.info("Saving team image to Amazon S3")
fileName = 'images/teams/'+str(teams[teamName])+'.jpg'
fileType = IPUtils.saveBase64EncodedImageToAmazonS3(metadata.XPath.Team_Image(team)[0].text, fileName, s3bucket)
else:
logger.info("Saving team image to file")
fileName = '/var/tmp/inplayrs/files/images/teams/'+str(teams[teamName])+'.jpg'
fileType = IPUtils.saveBase64EncodedImageToFile(metadata.XPath.Team_Image(team)[0].text, fileName)
if (fileType == None):
logger.warning("Invalid image found in tag, unable to save")
else:
logger.info("Successfully saved image fileType="+fileType+", fileName="+fileName)
else:
logger.info("No team image found")
# Cycle through the squad and inert into the player table if they do not already exist
for player in metadata.XPath.Team_Players(team):
processPlayer(player, teams[teamName])
# Get script name
scriptName = str(os.path.basename(__file__)).replace(".py", "")
# Parse command line arguments
parser = argparse.ArgumentParser()
parser.add_argument("env",
choices=['local', 'dev', 'prod'],
metavar="env",
help="Environment (local/dev/prod)")
parser.add_argument("-d", "--debug", help="Debug mode", action="store_true")
args = parser.parse_args()
# Get logger
loggingLevel = (logging.DEBUG if args.debug else logging.INFO)
logger = IPUtils.getLogger(scriptName, loggingLevel)
# load DB config and connect to DB
dbConfig = config.Connections.dbConfig.get(args.env)
db = pymysql.connect(host=dbConfig['host'],
port=dbConfig['port'],
user=dbConfig['user'],
passwd=dbConfig['pass'],
db=dbConfig['db'])
db.autocommit(1)
############################### GLOBAL VARIABLES ###############################
################################## FUNCTIONS ###################################