コード例 #1
0
 def __init__(self, queue, token, i,time):
           threading.Thread.__init__(self)
           self.queue = queue
           self.dbConn = DatabaseConnection()
           self.apiConn = APIConnection()
           self.apiConn.setToken(token)
           self.threadNumber = i
           self.episodeThreads = 15
           self.lastUpdatedTime = time
コード例 #2
0
class CkanOrganization():
    def __init__(self, ckan):
        self.api = APIConnection(ckan)

    def load_organization(self, id):
        try:
            self.organization = self.api.get_organization(id)
        except APIException as apiex:
            logging.error(apiex)
            raise OrganizationException("Problem to get organizarion " +
                                        str(id))

    def get_owner(self):
        return self.organization["id"]
コード例 #3
0
ファイル: CkanPackage.py プロジェクト: esgg/ckan-deliver
class CkanPackage():
    def __init__(self, api_address):
        self.api = APIConnection(api_address)

    def load_package(self, id):
        try:
            self.package = self.api.get_package(id)
        except APIException as apiex:
            logging.error(apiex)
            raise PackageException("Problem to get package "+str(id))

    def exists_package(self, id):
        try:
            return self.api.check_package(id)
        except APIException as apiex:
            logging.error(apiex)
            return False

    def update(self,id, package):
        self.api.update_package(id,package)

    def get_package(self):
        return self.package

    def delete_metadata(self, list_metadata):
        for metadata in list_metadata:
            try:
                del self.package[metadata]
            except Exception as ex:
                logging.error()

    def get_organization_id(self):
        return self.package["organization"]["name"]

    def add_metadata(self, metadata, value):
        self.package[metadata] = value

    def get_metadata(self, metadata):
        return self.package[metadata]

    def write_package(self, package):
        self.api.write_package(package)
コード例 #4
0
 def generateConnection(credentials, connection):
     apiconnection = APIConnection(credentials["costumer_key"], credentials["consumer_secret"],
                                   credentials["access_token"], credentials["access_token_secret"], connection)
     apiconnection.setConnection(apiconnection.getAccessToken(), apiconnection.getAccessTokenSecret(),
                                 apiconnection.getCustomerKey(), apiconnection.getCustomerSecret(), connection)
     return apiconnection.getConnection()
コード例 #5
0
def main(dayTimeSpan=1):
    file = open('settings.txt')
    userData = {}
    for line in file:
        splitLine = line.split('=')
        userData[splitLine[0]] = splitLine[1].rstrip()


    api = APIConnection()
    token = api.getToken(userData)

    processedSeriesCounter = 0
    processedSeries = set()
    updatedSeries = Queue()

    num_threads = 10
    threadList = []

    lastUpdatedTime = int(userData["previoustime"])
    startEpochDate = datetime.datetime(1970,1,1)
    loopTime = startEpochDate + datetime.timedelta(seconds=lastUpdatedTime)
    startTime = datetime.datetime.now()
    
    dayCounter = dayTimeSpan
    timeSpanDays = dayTimeSpan
    timeSpanSeconds = int(timeSpanDays * 86400)

    print loopTime.strftime("Starting at %d-%m-%Y %H:%M:%S")

    while loopTime <= startTime and dayCounter <= timeSpanDays:
        epochTime = (loopTime - startEpochDate).total_seconds()
        toTime = int(epochTime) + timeSpanSeconds # Adds one day to start time
        weeklyUpdates = api.getUpdatedSeries(int(epochTime),toTime)
        if weeklyUpdates:
            weeklyUpdatesCounter = 0
            for updatedSerie in weeklyUpdates:
                id = updatedSerie["id"]
                if id not in processedSeries:
                    updatedSeries.put((id,updatedSerie['lastUpdated']))
                    processedSeriesCounter += 1
                    processedSeries.add(id)
                weeklyUpdatesCounter += 1
        else:
            print "No series where updated during this timespan" 
        loopTime += datetime.timedelta(seconds=timeSpanSeconds)
        dayCounter += timeSpanDays
    
    print "Processing %s series." % (processedSeriesCounter)

    for i in range(num_threads):
        worker = processSerie(updatedSeries,token, i, lastUpdatedTime)
        worker.setDaemon(True)
        worker.start()
        threadList.append(worker)
    
    updatedSeries.join()

    for worker in threadList:
        worker.dbConn.db.close()

    elapsedTime = datetime.datetime.now() - startTime
    elapsedTimeMilliSeconds = elapsedTime.microseconds
    elapsedTimeTotalSeconds = elapsedTime.seconds
    elapsedTimeSeconds = elapsedTimeTotalSeconds % 60
    elapsedTimeMinutes = (elapsedTimeTotalSeconds - elapsedTimeSeconds) / 60

    if processedSeriesCounter > 0:
        if loopTime >= startTime:
            newTime = int((startTime - startEpochDate).total_seconds())
        else:
            newTime = int((loopTime - startEpochDate).total_seconds())
        updateSettings(newTime,userData)
        
    
    print "Updating info from %s series took %s:%s minutes.\n\n" % (processedSeriesCounter,elapsedTimeMinutes,elapsedTimeSeconds)
コード例 #6
0
class processSerie(threading.Thread):
    def __init__(self, queue, token, i,time):
              threading.Thread.__init__(self)
              self.queue = queue
              self.dbConn = DatabaseConnection()
              self.apiConn = APIConnection()
              self.apiConn.setToken(token)
              self.threadNumber = i
              self.episodeThreads = 15
              self.lastUpdatedTime = time

    def run(self):
        while True:
            updatedSerie = self.queue.get()
            
            id = updatedSerie[0]
            lastupdated = updatedSerie[1]
            if self.lastUpdatedTime < lastupdated:
                start = datetime.datetime.now()
                serieData = self.apiConn.getSerieData(id)
                if serieData:
                    firstCharsSerieName = serieData['seriesName'][:2] # Duplicates in the API start with ** so these should not be handled and updated.
                    if firstCharsSerieName != "**":
                        self.dbConn.updateSerieInfo(serieData)
                        part1 = (datetime.datetime.now() - start).seconds
                        episodes = self.apiConn.getUpdatedEpisodes(id)
                        part2 = ((datetime.datetime.now() - start).seconds) - part1 
                        EpisodeQueue = Queue()
                        episodeCounter = 0
                        if episodes:
                            distinctSeasons = self.apiConn.getDistinctSeasons(episodes)        
                            seasonInfo = self.dbConn.getSeasonInfo(distinctSeasons)
                            for episode in episodes['data']:
                                episode['seriesID'] = id
                                if episode['airedSeason'] and seasonInfo:
                                    episode['airedSeason'] = int(episode['airedSeason'])
                                    episode['seasonID'] = seasonInfo[episode['airedSeason']]
                                    EpisodeQueue.put(episode)
                                    episodeCounter += 1

                            print "[%s]Starting: %s, %s episodes.\r" % (datetime.datetime.now().strftime('%H:%M:%S'),serieData['seriesName'],episodeCounter)
                            for i in range(self.episodeThreads):
                                t = threading.Thread(target=self.getEpisode,args=(EpisodeQueue,i))
                                t.daemon = True
                                t.start()
                                # print "%s: Processing %s S%sE%s - %s" % (self.threadNumber,serieData['seriesName'],episode['airedSeason'],episode['airedEpisodeNumber'],episode['episodeName'].encode('utf-8'))
                            
                                # detailedEpisode = self.apiConn.getDetailedEpisodeInfo(episode['id'])
                                # self.db.updateEpisodeInfo(detailedEpisode)
                        EpisodeQueue.join()
                        end = (datetime.datetime.now() - start).seconds
                        print "[%s]Finished: %s, %s episodes.\r" % (datetime.datetime.now().strftime('%H:%M:%S'),serieData['seriesName'],episodeCounter)
            self.queue.task_done()

    def getEpisode(self,episodeQueue,threadID):
        while True:
            episode = episodeQueue.get()
            detailedEpisode = self.apiConn.getDetailedEpisodeInfo(episode['id'])
            if detailedEpisode:
                detailedEpisode['seasonID'] = episode['seasonID']
                newdb = DatabaseConnection()
                newdb.updateEpisodeInfo(detailedEpisode)
                newdb.db.close()
                episodeQueue.task_done()
        return None
コード例 #7
0
ファイル: CkanPackage.py プロジェクト: esgg/ckan-deliver
 def __init__(self, api_address):
     self.api = APIConnection(api_address)
コード例 #8
0
 def __init__(self, ckan):
     self.api = APIConnection(ckan)