def __init__(self, queue, token, i,time):
threading.Thread.__init__(self)
self.queue = queue
self.dbConn = DatabaseConnection()
self.apiConn = APIConnection()
self.apiConn.setToken(token)
self.threadNumber = i
self.episodeThreads = 15
self.lastUpdatedTime = time
class CkanOrganization():
def __init__(self, ckan):
self.api = APIConnection(ckan)
def load_organization(self, id):
try:
self.organization = self.api.get_organization(id)
except APIException as apiex:
logging.error(apiex)
raise OrganizationException("Problem to get organizarion " +
str(id))
def get_owner(self):
return self.organization["id"]
class CkanPackage():
def __init__(self, api_address):
self.api = APIConnection(api_address)
def load_package(self, id):
try:
self.package = self.api.get_package(id)
except APIException as apiex:
logging.error(apiex)
raise PackageException("Problem to get package "+str(id))
def exists_package(self, id):
try:
return self.api.check_package(id)
except APIException as apiex:
logging.error(apiex)
return False
def update(self,id, package):
self.api.update_package(id,package)
def get_package(self):
return self.package
def delete_metadata(self, list_metadata):
for metadata in list_metadata:
try:
del self.package[metadata]
except Exception as ex:
logging.error()
def get_organization_id(self):
return self.package["organization"]["name"]
def add_metadata(self, metadata, value):
self.package[metadata] = value
def get_metadata(self, metadata):
return self.package[metadata]
def write_package(self, package):
self.api.write_package(package)
def generateConnection(credentials, connection):
apiconnection = APIConnection(credentials["costumer_key"], credentials["consumer_secret"],
credentials["access_token"], credentials["access_token_secret"], connection)
apiconnection.setConnection(apiconnection.getAccessToken(), apiconnection.getAccessTokenSecret(),
apiconnection.getCustomerKey(), apiconnection.getCustomerSecret(), connection)
return apiconnection.getConnection()
def main(dayTimeSpan=1):
file = open('settings.txt')
userData = {}
for line in file:
splitLine = line.split('=')
userData[splitLine[0]] = splitLine[1].rstrip()
api = APIConnection()
token = api.getToken(userData)
processedSeriesCounter = 0
processedSeries = set()
updatedSeries = Queue()
num_threads = 10
threadList = []
lastUpdatedTime = int(userData["previoustime"])
startEpochDate = datetime.datetime(1970,1,1)
loopTime = startEpochDate + datetime.timedelta(seconds=lastUpdatedTime)
startTime = datetime.datetime.now()
dayCounter = dayTimeSpan
timeSpanDays = dayTimeSpan
timeSpanSeconds = int(timeSpanDays * 86400)
print loopTime.strftime("Starting at %d-%m-%Y %H:%M:%S")
while loopTime <= startTime and dayCounter <= timeSpanDays:
epochTime = (loopTime - startEpochDate).total_seconds()
toTime = int(epochTime) + timeSpanSeconds # Adds one day to start time
weeklyUpdates = api.getUpdatedSeries(int(epochTime),toTime)
if weeklyUpdates:
weeklyUpdatesCounter = 0
for updatedSerie in weeklyUpdates:
id = updatedSerie["id"]
if id not in processedSeries:
updatedSeries.put((id,updatedSerie['lastUpdated']))
processedSeriesCounter += 1
processedSeries.add(id)
weeklyUpdatesCounter += 1
else:
print "No series where updated during this timespan"
loopTime += datetime.timedelta(seconds=timeSpanSeconds)
dayCounter += timeSpanDays
print "Processing %s series." % (processedSeriesCounter)
for i in range(num_threads):
worker = processSerie(updatedSeries,token, i, lastUpdatedTime)
worker.setDaemon(True)
worker.start()
threadList.append(worker)
updatedSeries.join()
for worker in threadList:
worker.dbConn.db.close()
elapsedTime = datetime.datetime.now() - startTime
elapsedTimeMilliSeconds = elapsedTime.microseconds
elapsedTimeTotalSeconds = elapsedTime.seconds
elapsedTimeSeconds = elapsedTimeTotalSeconds % 60
elapsedTimeMinutes = (elapsedTimeTotalSeconds - elapsedTimeSeconds) / 60
if processedSeriesCounter > 0:
if loopTime >= startTime:
newTime = int((startTime - startEpochDate).total_seconds())
else:
newTime = int((loopTime - startEpochDate).total_seconds())
updateSettings(newTime,userData)
print "Updating info from %s series took %s:%s minutes.\n\n" % (processedSeriesCounter,elapsedTimeMinutes,elapsedTimeSeconds)
class processSerie(threading.Thread):
def __init__(self, queue, token, i,time):
threading.Thread.__init__(self)
self.queue = queue
self.dbConn = DatabaseConnection()
self.apiConn = APIConnection()
self.apiConn.setToken(token)
self.threadNumber = i
self.episodeThreads = 15
self.lastUpdatedTime = time
def run(self):
while True:
updatedSerie = self.queue.get()
id = updatedSerie[0]
lastupdated = updatedSerie[1]
if self.lastUpdatedTime < lastupdated:
start = datetime.datetime.now()
serieData = self.apiConn.getSerieData(id)
if serieData:
firstCharsSerieName = serieData['seriesName'][:2] # Duplicates in the API start with ** so these should not be handled and updated.
if firstCharsSerieName != "**":
self.dbConn.updateSerieInfo(serieData)
part1 = (datetime.datetime.now() - start).seconds
episodes = self.apiConn.getUpdatedEpisodes(id)
part2 = ((datetime.datetime.now() - start).seconds) - part1
EpisodeQueue = Queue()
episodeCounter = 0
if episodes:
distinctSeasons = self.apiConn.getDistinctSeasons(episodes)
seasonInfo = self.dbConn.getSeasonInfo(distinctSeasons)
for episode in episodes['data']:
episode['seriesID'] = id
if episode['airedSeason'] and seasonInfo:
episode['airedSeason'] = int(episode['airedSeason'])
episode['seasonID'] = seasonInfo[episode['airedSeason']]
EpisodeQueue.put(episode)
episodeCounter += 1
print "[%s]Starting: %s, %s episodes.\r" % (datetime.datetime.now().strftime('%H:%M:%S'),serieData['seriesName'],episodeCounter)
for i in range(self.episodeThreads):
t = threading.Thread(target=self.getEpisode,args=(EpisodeQueue,i))
t.daemon = True
t.start()
# print "%s: Processing %s S%sE%s - %s" % (self.threadNumber,serieData['seriesName'],episode['airedSeason'],episode['airedEpisodeNumber'],episode['episodeName'].encode('utf-8'))
# detailedEpisode = self.apiConn.getDetailedEpisodeInfo(episode['id'])
# self.db.updateEpisodeInfo(detailedEpisode)
EpisodeQueue.join()
end = (datetime.datetime.now() - start).seconds
print "[%s]Finished: %s, %s episodes.\r" % (datetime.datetime.now().strftime('%H:%M:%S'),serieData['seriesName'],episodeCounter)
self.queue.task_done()
def getEpisode(self,episodeQueue,threadID):
while True:
episode = episodeQueue.get()
detailedEpisode = self.apiConn.getDetailedEpisodeInfo(episode['id'])
if detailedEpisode:
detailedEpisode['seasonID'] = episode['seasonID']
newdb = DatabaseConnection()
newdb.updateEpisodeInfo(detailedEpisode)
newdb.db.close()
episodeQueue.task_done()
return None
def __init__(self, api_address):
self.api = APIConnection(api_address)
def __init__(self, ckan):
self.api = APIConnection(ckan)