def testNotTargetUser(self, mock2, mock1):
self.bucket = Bucket()
mock1.return_value = [S3Object()]
mock2.return_value = json.loads(jsonData_not_terget_user)
self.bucket.append('AWSLogs/210811600188/CloudTrail/us-east-1/2018/03/07')
df = self.bucket.data()
self.assertEqual(len(df.index), 0)
def ProcessBucket(response, s3Client, s3Resource, params):
result = {}
for bucket in response['Buckets']:
# checks if the filter flag exists and if the filter flag is a valid name
if (params['f'] and IsBucketNameValid(
bucket['Name'], params['f'])) or (params['f'] is None):
currentBucket = Bucket() # instantiate new Bucket
currentBucket.name = bucket['Name'] # set name attribute
currentBucket.region = s3Client.get_bucket_location(
Bucket=currentBucket.name)[
'LocationConstraint'] # set region attribute
# set creation date attribute
currentBucket.creationDate = bucket['CreationDate'].ctime()
# checks if there are ANY object level data requested in the argument
if (any(param == True
for param in params['objectParams'].values())):
ProcessBucketObjects(currentBucket, bucket,
params['objectParams'], s3Client,
s3Resource)
# insert bucket, all relevant data should be populated
# check to see whether the region is already in result if so group them together, otherwise instantiate a list within a dict like:
# result = {Key=RegionName, Val = [This is a list of buckets, append current one]}
if currentBucket.region in result.keys():
result[currentBucket.region].append(currentBucket)
else:
result[currentBucket.region] = [currentBucket]
return result
def testSuccess(self, mock2, mock1):
self.bucket = Bucket()
mock1.return_value = [S3Object()]
mock2.return_value = json.loads(jsonData)
self.bucket.append('AWSLogs/210811600188/CloudTrail/us-east-1/2018/03/07')
df = self.bucket.data()
self.assertEqual(len(df.index), 1)
self.assertEqual(df['userName'][0], Bucket.TARGET_USER)
self.assertEqual(df['eventName'][0], "DescribeInstanceStatus")
self.assertEqual(df['errorCode'][0], None)
self.assertEqual(df['errorMessage'][0], None)
self.assertEqual(df['date'][0], "2018-03-06T23:58:55Z")
def insert(self, key, value):
if self.root == None:
new_node = Node(key, value)
new_node.is_red = False
self.root = new_node
self.count += 1
left_child = Node(-1, -1) # this is an empty bucket
left_child.is_red = False
self.root.left = left_child
left_child.parent = self.root
left_child.has_bucket = True
left_child.bucket = Bucket(None, None, left_child)
left_child.bucket.fixing_pointer = self.nil
self.leftmost_bucket = left_child.bucket
right_child = Node(-1, -1)
right_child.is_red = False
self.root.right = right_child
right_child.parent = self.root
right_child.has_bucket = True
right_child.bucket = Bucket(left_child.bucket, None, right_child)
right_child.bucket.fixing_pointer = self.nil
left_child.bucket.right_bucket = right_child.bucket
self.global_bucket = self.leftmost_bucket
else:
# binary search till we find the bucket
temp_node = self.root
while temp_node.has_bucket == False:
if key >= temp_node.key:
temp_node = temp_node.right
else:
temp_node = temp_node.left
bucket = temp_node.bucket
bucket.insert_item(key, value)
self.fixup(bucket)
self.fixup(bucket)
# check for splitting case
if bucket.count > (ceil(2 * self.H)) - 10 and bucket.count > 20:
self.split(temp_node)
self.post_operation()
self.update_H()
return
class BucketPlayer:
def __init__(self, devices):
self.bucket = Bucket(devices)
self.player = Player()
self.bucket_state = BucketState.UNKNOWN
def run_detection(self):
new_bucket_state = self.bucket.detect()
if self.bucket_state != new_bucket_state:
if new_bucket_state is BucketState.ONE_IN_FROM_LEFT:
logging.info('Detected person coming from left. Start playing video.')
self.player.play_video(0)
elif new_bucket_state is BucketState.ONE_IN_FROM_RIGHT:
logging.info('Detected person coming from right. Start playing video.')
self.player.play_video(1)
elif new_bucket_state is BucketState.TWO_IN:
logging.info('Detected person coming from both sides. Start playing video.')
self.player.play_video(2)
else:
logging.info(
'Current state {} does not match any of the rules. Stopping video...'.format(new_bucket_state))
self.player.stop_playing()
self.bucket_state = new_bucket_state
def main():
global password_file
global journal_users
global port
logging.basicConfig(level=logging.INFO)
if 'BUCKET_DIR_BASE' not in environ and 'S3_BUCKET_BASE' not in environ:
raise ValueError(
"Need BUCKET_DIR_BASE or S3_BUCKET_BASE in environment")
if 'USER_TZ' in environ:
# Running in AWS, so need to account for TZ.
Entry.user_tz = environ['USER_TZ']
if 'PASSWORD_FILE' in environ:
password_file = environ['PASSWORD_FILE']
port = 5000
if 'PORT' in environ:
port = int(environ['PORT'])
logging.info("Set port to {}".format(port))
debug = False
if 'DEBUG' in environ:
debug = True
journal_users = User.users_from_password_file(password_file)
for user in journal_users:
user_buckets[user] = Bucket.bucket_factory(user)
app.secret_key = urandom(12)
app.run(debug=debug, port=port)
def chaining_with_hash3(self, value, size):
"""
:param value: Customer obj
:param size: maximum size of list
:return:
"""
hash = self.hash_func3(int(value.customer_id.replace('-', '')), size)
if len(self.table.array) < hash:
linked_list = None
else:
linked_list = self.table.array[hash]
if linked_list is None:
linked_list = Bucket(k=hash)
linked_list.Append(value)
self.table.dynamic_set(key=hash, value=linked_list)
else:
linked_list.Append(value)
self.collisions += 1
def start(self):
checks = self.__checks()
if checks:
self.serverSocket.listen()
while True:
print(
f"[CURRENT ACTIVE CONNECTIONS] {threading.activeCount() - 1}"
)
print(f"[LISTENING]...")
conn, clientAddress = self.serverSocket.accept()
buckets = Bucket(self.bucketsPath)
thread = threading.Thread(target=self.__clientHandler,
args=(conn, clientAddress, buckets))
thread.start()
def getGoodwordsBucketList(wordsFileName):
with codecs.open(wordsFileName, "r", "utf-8") as dictFile:
buckets = []
for line in dictFile:
goodWord = line[:-1]
bucket = Bucket()
bucket.word = goodWord
bucket.sortedWord = getArrangedLetters(goodWord)
bucket.wordLength = len(goodWord)
bucket.bucket = makeBucketsLetters(goodWord)
buckets.append(bucket)
return (buckets)
class BucketTest(unittest.TestCase):
##
## Name: setUp
## Description: Setup fixture for the BucketTest class
##
## Parameters:
## None
##
@patch('Bucket.Bucket.__initData__')
def setUp(self, mock1):
mock1.return_value = None
##
## Name: testSuccess
## Description: Test the success leg for the Bucket class
##
## Parameters:
## None
##
@patch('Bucket.Bucket.__listBucketObjects__')
@patch('Bucket.Bucket.__loadJsonData__')
def testSuccess(self, mock2, mock1):
self.bucket = Bucket()
mock1.return_value = [S3Object()]
mock2.return_value = json.loads(jsonData)
self.bucket.append('AWSLogs/210811600188/CloudTrail/us-east-1/2018/03/07')
df = self.bucket.data()
self.assertEqual(len(df.index), 1)
self.assertEqual(df['userName'][0], Bucket.TARGET_USER)
self.assertEqual(df['eventName'][0], "DescribeInstanceStatus")
self.assertEqual(df['errorCode'][0], None)
self.assertEqual(df['errorMessage'][0], None)
self.assertEqual(df['date'][0], "2018-03-06T23:58:55Z")
##
## Name: testNotTargetUser
## Description: Test the case where it is a log entry not associated with the target user.
##
## Parameters:
## None
##
@patch('Bucket.Bucket.__listBucketObjects__')
@patch('Bucket.Bucket.__loadJsonData__')
def testNotTargetUser(self, mock2, mock1):
self.bucket = Bucket()
mock1.return_value = [S3Object()]
mock2.return_value = json.loads(jsonData_not_terget_user)
self.bucket.append('AWSLogs/210811600188/CloudTrail/us-east-1/2018/03/07')
df = self.bucket.data()
self.assertEqual(len(df.index), 0)
def buckets_on_image(self, image):
bucket_markers = self.bucket_markers(image)
if not bucket_markers:
return []
corrected_bucket_numbers = self.corrected_bucket_numbers(
bucket_markers)
unique_bucket_numbers = self.unique_bucket_numbers(
corrected_bucket_numbers)
if unique_bucket_numbers == [""]:
return []
bucket_borders = self.bucket_borders(bucket_markers, image)
buckets_on_image = []
for i in range(0, len(bucket_borders) - 1):
actual_bucket = Bucket(start=bucket_borders[i],
end=bucket_borders[i + 1],
bucket_number=unique_bucket_numbers[i])
buckets_on_image.append(actual_bucket)
return buckets_on_image
def main(argv):
# Command line argument parsing
c_args = argparse.ArgumentParser(description='Explore Cloud Trail Logs')
c_args.add_argument('--region',
help='AWS region',
default="us-east-1",
dest="region",
required=False)
c_args.add_argument('--start',
help='Start date in month/day/year format',
default=None)
c_args.add_argument('--end',
help='End date in month/day/year format',
default=None)
c_args.add_argument('--format',
help='Output format: TEXT | HTML. Case insensitive',
default="TEXT",
dest="format",
required=False)
cli_args = c_args.parse_args()
region = cli_args.region
start = cli_args.start
end = cli_args.end
format = cli_args.format
# Initializations
formatter = ReportFormatterFactory.create(
ReportFormatterType.fromString(format))
bucket = Bucket()
# Validity checks
if None == start:
print "Error: missing parameter - start"
print cli_args.echo
exit(-1)
if not re.match(r'[0-9]{1,2}/[0-9]{1,2}/[0-9]{4}', start):
print "Error: incorrect date format - start: " + start
print cli_args.echo
exit(-1)
if None == end:
print "Error: missing parameter - end"
print cli_args.echo
exit(-1)
if not re.match(r'[0-9]{1,2}/[0-9]{1,2}/[0-9]{4}', end):
print "Error: incorrect date format - end: " + end
print cli_args.echo
exit(-1)
if None == formatter:
print "Error: invalid format type - format: " + format
print cli_args.echo
exit(-1)
# Main processing loop based on range of dates
start = datetime.datetime.strptime(start, "%m/%d/%Y")
end = datetime.datetime.strptime(end, "%m/%d/%Y")
for date in [
start + datetime.timedelta(days=x)
for x in range(0, (end - start).days)
]:
date_arr = "{:%m/%d/%Y}".format(date).split('/')
if len(date_arr) != 3:
print "Error: incorrect format - date: " + date
exit(-1)
month = int(date_arr[-3])
day = int(date_arr[-2])
year = int(date_arr[-1])
bucket_path = CLOUDTRAIL_BUCKET_PATH_FMT.format(
region, year, month, day)
bucket.append(bucket_path)
# Output report
df = bucket.data()
formatter.printHeader()
formatter.eventSummary(df)
formatter.errorSummary(df)
formatter.errorsByDateSummary(df)
formatter.printTrailer()
class RequestFactory:
def __init__(self, core):
self.core = core
self.bucket = Bucket()
self.updateBucket()
self.core.evm.listenTo("config:changed", self.updateConfig)
def getURL(self, *args, **kwargs):
""" see HTTPRequest for argument list """
h = DefaultRequest(self.getConfig())
try:
rep = h.load(*args, **kwargs)
finally:
h.close()
return rep
########## old api methods above
def getRequest(self, context=None, klass=DefaultRequest):
""" Creates a request with new or given context """
# also accepts the context class directly
if isinstance(context, klass.CONTEXT_CLASS):
return klass(self.getConfig(), context)
elif context:
return klass(*context)
else:
return klass(self.getConfig())
def getDownloadRequest(self, request=None, klass=DefaultDownload):
""" Instantiates a instance for downloading """
# TODO: load with plugin manager
return klass(self.bucket, request)
def getInterface(self):
return self.core.config["download"]["interface"]
def getProxies(self):
""" returns a proxy list for the request classes """
if not self.core.config["proxy"]["proxy"]:
return {}
else:
type = "http"
setting = self.core.config["proxy"]["type"].lower()
if setting == "socks4":
type = "socks4"
elif setting == "socks5":
type = "socks5"
username = None
if self.core.config["proxy"]["username"] and self.core.config["proxy"]["username"].lower() != "none":
username = self.core.config["proxy"]["username"]
pw = None
if self.core.config["proxy"]["password"] and self.core.config["proxy"]["password"].lower() != "none":
pw = self.core.config["proxy"]["password"]
return {
"type": type,
"address": self.core.config["proxy"]["address"],
"port": self.core.config["proxy"]["port"],
"username": username,
"password": pw,
}
def updateConfig(self, section, option, value):
""" Updates the bucket when a config value changed """
if option in ("limit_speed", "max_speed"):
self.updateBucket()
def getConfig(self):
"""returns options needed for pycurl"""
return {"interface": self.getInterface(),
"proxies": self.getProxies(),
"ipv6": self.core.config["download"]["ipv6"]}
def updateBucket(self):
""" set values in the bucket according to settings"""
if not self.core.config["download"]["limit_speed"]:
self.bucket.setRate(-1)
else:
self.bucket.setRate(self.core.config["download"]["max_speed"] * 1024)
class RequestFactory():
def __init__(self, core):
self.lock = Lock()
self.core = core
self.bucket = Bucket()
self.updateBucket()
@property
def iface(self):
return self.core.config["download"]["interface"]
def getRequest(self, pluginName, cj=None):
req = Browser(self.bucket, self.getOptions())
if cj:
req.setCookieJar(cj)
else:
req.setCookieJar(CookieJar(pluginName))
return req
def getHTTPRequest(self, **kwargs):
""" returns a http request, don't forget to close it ! """
options = self.getOptions()
options.update(kwargs) # submit kwargs as additional options
return HTTPRequest(CookieJar(None), options)
def getURL(self, *args, **kwargs):
""" see HTTPRequest for argument list """
h = HTTPRequest(None, self.getOptions())
try:
rep = h.load(*args, **kwargs)
finally:
h.close()
return rep
def openCookieJar(self, pluginname):
"""Create new CookieJar"""
return CookieJar(pluginname)
def getProxies(self):
""" returns a proxy list for the request classes """
if not self.core.config["proxy"]["proxy"]:
return {}
else:
type = "http"
setting = self.core.config["proxy"]["type"].lower()
if setting == "socks4": type = "socks4"
elif setting == "socks5": type = "socks5"
username = None
if self.core.config["proxy"]["username"] and self.core.config[
"proxy"]["username"].lower() != "none":
username = self.core.config["proxy"]["username"]
pw = None
if self.core.config["proxy"]["password"] and self.core.config[
"proxy"]["password"].lower() != "none":
pw = self.core.config["proxy"]["password"]
return {
"type": type,
"address": self.core.config["proxy"]["address"],
"port": self.core.config["proxy"]["port"],
"username": username,
"password": pw,
}
def getOptions(self):
"""returns options needed for pycurl"""
return {
"interface": self.iface,
"proxies": self.getProxies(),
"ipv6": self.core.config["download"]["ipv6"]
}
def updateBucket(self):
""" set values in the bucket according to settings"""
if not self.core.config["download"]["limit_speed"]:
self.bucket.setRate(-1)
else:
self.bucket.setRate(self.core.config["download"]["max_speed"] *
1024)
chunk.close()
def close(self):
""" cleanup """
for chunk in self.chunks:
self.closeChunk(chunk)
self.chunks = []
if hasattr(self, "m"):
self.m.close()
del self.m
if hasattr(self, "cj"):
del self.cj
if hasattr(self, "info"):
del self.info
if __name__ == "__main__":
url = "http://speedtest.netcologne.de/test_100mb.bin"
from Bucket import Bucket
bucket = Bucket()
bucket.setRate(200 * 1024)
bucket = None
print "starting"
dwnld = HTTPDownload(url, "test_100mb.bin", bucket=bucket)
dwnld.download(chunks=3, resume=True)
def __init__(self, core):
self.core = core
self.bucket = Bucket()
self.updateBucket()
self.core.evm.listenTo("config:changed", self.updateConfig)
class RequestFactory():
def __init__(self, core):
self.lock = Lock()
self.core = core
self.bucket = Bucket()
self.updateBucket()
self.cookiejars = {}
def iface(self):
return self.core.config["download"]["interface"]
def getRequest(self, pluginName, account=None, type="HTTP"):
self.lock.acquire()
if type == "XDCC":
return XDCCRequest(proxies=self.getProxies())
req = Browser(self.bucket, self.getOptions())
if account:
cj = self.getCookieJar(pluginName, account)
req.setCookieJar(cj)
else:
req.setCookieJar(CookieJar(pluginName))
self.lock.release()
return req
def getHTTPRequest(self, **kwargs):
""" returns a http request, dont forget to close it ! """
options = self.getOptions()
options.update(kwargs) # submit kwargs as additional options
return HTTPRequest(CookieJar(None), options)
def getURL(self, *args, **kwargs):
""" see HTTPRequest for argument list """
h = HTTPRequest(None, self.getOptions())
try:
rep = h.load(*args, **kwargs)
finally:
h.close()
return rep
def getCookieJar(self, pluginName, account=None):
if (pluginName, account) in self.cookiejars:
return self.cookiejars[(pluginName, account)]
cj = CookieJar(pluginName, account)
self.cookiejars[(pluginName, account)] = cj
return cj
def getProxies(self):
""" returns a proxy list for the request classes """
if not self.core.config["proxy"]["proxy"]:
return {}
else:
type = "http"
setting = self.core.config["proxy"]["type"].lower()
if setting == "socks4": type = "socks4"
elif setting == "socks5": type = "socks5"
username = None
if self.core.config["proxy"]["username"] and self.core.config["proxy"]["username"].lower() != "none":
username = self.core.config["proxy"]["username"]
pw = None
if self.core.config["proxy"]["password"] and self.core.config["proxy"]["password"].lower() != "none":
pw = self.core.config["proxy"]["password"]
return {
"type": type,
"address": self.core.config["proxy"]["address"],
"port": self.core.config["proxy"]["port"],
"username": username,
"password": pw,
}
def getOptions(self):
"""returns options needed for pycurl"""
return {"interface": self.iface(),
"proxies": self.getProxies(),
"ipv6": self.core.config["download"]["ipv6"]}
def updateBucket(self):
""" set values in the bucket according to settings"""
if not self.core.config["download"]["limit_speed"]:
self.bucket.setRate(-1)
else:
self.bucket.setRate(self.core.config["download"]["max_speed"] * 1024)
def __init__(self, core):
self.core = core
self.bucket = Bucket()
self.updateBucket()
def __init__(self, core):
self.core = core
self.bucket = Bucket()
self.updateBucket()
def __init__(self, core):
self.core = core
self.bucket = Bucket()
self.updateBucket()
self.core.evm.listenTo("config:changed", self.updateConfig)
def __init__(self, core):
self.lock = Lock()
self.core = core
self.bucket = Bucket()
self.updateBucket()
self.cookiejars = {}
from store_device_certificates_errors import StorageFailed
# Environment variables
bucket_name = os.environ['IOT_DEVICES_CERTS_BUCKET']
flow_finished_topic = os.environ['IOT_DEVICE_REGISTRATION_FINISHED_TOPIC']
# AWS services clients
s3_client = boto3.client('s3')
sns_client = boto3.client('sns')
# Logger setup
logger = logging.getLogger()
logger.setLevel(logging.INFO)
# Core classes
bucket = Bucket(s3_client=s3_client)
queue = QueueHelper(sns_client=sns_client)
certificates_storage = DeviceCertificateStorage(bucket=bucket)
def store_device_certificates(event, context):
logger.info('## EVENT RECEIVED')
logger.info(event)
certificates = queue.decode_message(event=event)
try:
certificates_storage.store_certificates(certificates, bucket_name)
logger.info('## CERTIFICATES STORED SUCCESSFULY')
logger.info(certificates)
class RequestFactory:
def __init__(self, core):
self.core = core
self.bucket = Bucket()
self.updateBucket()
def getURL(self, *args, **kwargs):
""" see HTTPRequest for argument list """
h = DefaultRequest(self.getConfig())
try:
rep = h.load(*args, **kwargs)
finally:
h.close()
return rep
########## old api methods above
def getRequest(self, context=None, klass=DefaultRequest):
""" Creates a request with new or given context """
# also accepts the context class directly
if isinstance(context, klass.CONTEXT_CLASS):
return klass(self.getConfig(), context)
elif context:
return klass(*context)
else:
return klass(self.getConfig())
def getDownloadRequest(self, request=None, klass=DefaultDownload):
""" Instantiates a instance for downloading """
# TODO: load with plugin manager
return klass(self.bucket, request)
def getInterface(self):
return self.core.config["download"]["interface"]
def getProxies(self):
""" returns a proxy list for the request classes """
if not self.core.config["proxy"]["proxy"]:
return {}
else:
type = "http"
setting = self.core.config["proxy"]["type"].lower()
if setting == "socks4":
type = "socks4"
elif setting == "socks5":
type = "socks5"
username = None
if self.core.config["proxy"]["username"] and self.core.config[
"proxy"]["username"].lower() != "none":
username = self.core.config["proxy"]["username"]
pw = None
if self.core.config["proxy"]["password"] and self.core.config[
"proxy"]["password"].lower() != "none":
pw = self.core.config["proxy"]["password"]
return {
"type": type,
"address": self.core.config["proxy"]["address"],
"port": self.core.config["proxy"]["port"],
"username": username,
"password": pw,
}
def getConfig(self):
"""returns options needed for pycurl"""
return {
"interface": self.getInterface(),
"proxies": self.getProxies(),
"ipv6": self.core.config["download"]["ipv6"]
}
def updateBucket(self):
""" set values in the bucket according to settings"""
if not self.core.config["download"]["limit_speed"]:
self.bucket.setRate(-1)
else:
self.bucket.setRate(self.core.config["download"]["max_speed"] *
1024)
class methods:
def __init__(self, fps, clockObject, surface, font, bars, windowsize):
self.fps = fps
self.clock = clockObject
self.surface = surface
self.font = font
self.bars = bars
self.windowsize = windowsize
def get_array(self, length, mode=0):
arr = list(range(length))
if not mode:
random.shuffle(arr)
elif mode == 2:
arr = arr[::-1]
elif mode == 3:
for i in range(length - 1):
if random.randint(0, 10) < 8:
tmp = random.randint(4, 15)
try:
arr[i], arr[i + tmp] = arr[i + tmp], arr[i]
except:
pass
return arr
def setup(self, length, mode=0):
self.array = self.get_array(length, mode)
self.display = Display(self.windowsize[0] / length, self.windowsize,
self.surface, self.font)
self.accesses = 0
self.comparisons = 0
setattr(self.display, "bars", self.bars)
bubble = lambda self: Bubble(self.array, self.display, self.clock, self.fps
).main()
quicksort = lambda self: Quicksort(self.array, self.display, self.clock,
self.fps).main()
selection = lambda self: Selection(self.array, self.display, self.clock,
self.fps).main()
cocktail = lambda self: Cocktail(self.array, self.display, self.clock, self
.fps).main()
bogo = lambda self: Bogo(self.array, self.display, self.clock, self.fps
).main()
oddeven = lambda self: Oddeven(self.array, self.display, self.clock, self.
fps).main()
shell = lambda self: Shell(self.array, self.display, self.clock, self.fps
).main()
comb = lambda self: Comb(self.array, self.display, self.clock, self.fps
).main()
insertion = lambda self: Insertion(self.array, self.display, self.clock,
self.fps).main()
mergetd = lambda self: MergeTD(self.array, self.display, self.clock, self.
fps).main()
radixlsd = lambda self: RadixLSD(self.array, self.display, self.clock, self
.fps).main()
counting = lambda self: Counting(self.array, self.display, self.clock, self
.fps).main()
cycle = lambda self: Cycle(self.array, self.display, self.clock, self.fps
).main()
heap = lambda self: Heap(self.array, self.display, self.clock, self.fps
).main()
circle = lambda self: Circle(self.array, self.display, self.clock, self.fps
).main()
gnome = lambda self: Gnome(self.array, self.display, self.clock, self.fps
).main()
binaryinsertion = lambda self: BinaryInsertion(
self.array, self.display, self.clock, self.fps).main()
pancake = lambda self: Pancake(self.array, self.display, self.clock, self.
fps).main()
permutation = lambda self: Permutation(self.array, self.display, self.
clock, self.fps).main()
strand = lambda self: Strand(self.array, self.display, self.clock, self.fps
).main()
bucket = lambda self: Bucket(self.array, self.display, self.clock, self.fps
).main()
minmax = lambda self: MinMax(self.array, self.display, self.clock, self.fps
).main()
mergebu = lambda self: MergeBU(self.array, self.display, self.clock, self.
fps).main()
bitonic = lambda self: Bitonic(self.array, self.display, self.clock, self.
fps).main()
stooge = lambda self: Stooge(self.array, self.display, self.clock, self.fps
).main()
smooth = lambda self: Smooth(self.array, self.display, self.clock, self.fps
).main()
quick3 = lambda self: Quick3(self.array, self.display, self.clock, self.fps
).main()
# if(statistic:dcIdentifier is in our article:ingestDate dictionary):
# determine the correct bucket depending on article:ingestDate and statistic:dlDate
# addDownloads to that bucket for this statistic (make a bucket if necessary)
statistics = db.statistics.find()
for statistic in statistics:
if (statistic['dcIdentifier'] in ingestDates):
ingestDateChunks = ingestDates[statistic['dcIdentifier']].split('/')
ingestDate = datetime.date(int(ingestDateChunks[2]),
int(ingestDateChunks[0]),
int(ingestDateChunks[1]))
dlDate = statistic['dlDate'].date()
age = int(math.ceil(float((dlDate - ingestDate).days) / (356 / 12)))
#print "ingested {}, dlDate {}, age of {} mo".format(ingestDate, dlDate, age)
if (ageBuckets.get(age) == None):
ageBuckets[age] = Bucket(age)
ageBuckets[age].addDownloads(statistic['downloads'])
# now we have full buckets
avgDownloadsFileName = '../Web/stats/avgDownloads.txt'
avgDownloadsFile = open(avgDownloadsFileName, 'w')
for bucket in ageBuckets.keys():
#pprint(ageBuckets[bucket])
avgDownloadsFile.write('{}\t{}\n'.format(
ageBuckets[bucket].age,
round(
float(ageBuckets[bucket].downloads) / ageBuckets[bucket].articles,
2)))
avgDownloadsFile.close()
def __init__(self, devices):
self.bucket = Bucket(devices)
self.player = Player()
self.bucket_state = BucketState.UNKNOWN
class SetBucketUuid(object):
def process_request(self, req, resp):
if not (req.method == 'PUT' and req.path == '/'):
uuid = self.getBucketUuid(req)
req.context['bucketUuid'] = uuid
def getBucketUuid(self, req):
bucketName = self.getBucketName(req)
# get bucket uuid from db
bk = session.query(Bucket).filter(Bucket.name == bucketName).first()
return bk.uuid
def getBucketName(self, req):
# host = req.get_header('Host')
# bucketName = host.split('.')[0]
return req.get_header('BucketName')
app = falcon.API(middleware=[
AuthZCheck(),
SetBucketUuid(),
])
bucket = Bucket()
obj = Object()
app.add_route('/', bucket)
app.add_route('/{name}', obj)
def __init__(self, core):
self.lock = Lock()
self.core = core
self.bucket = Bucket()
self.updateBucket()
self.cookiejars = {}
from argparse import ArgumentParser
from time import mktime, strptime
from Entry import Entry
from Bucket import Bucket
parser = ArgumentParser()
parser.add_argument('--user', action='store', required=True)
parser.add_argument('--input_file', action='store', required=True)
args = parser.parse_args()
bucket = Bucket.bucket_factory(args.user)
with open(args.input_file, 'r') as input_file:
file_content = input_file.read()
chunks = file_content.split('-' * 80)
for chunk in chunks:
lines = chunk.split('\n')
while lines and not lines[0]:
lines.pop(0)
while lines and not lines[-1]:
lines.pop()
if not lines:
continue
struct_time = strptime(lines[0])
timestamp = mktime(struct_time)
text = '\n'.join(lines[2:])
print(m.find(6))
except NotFoundException:
print("Item not found")
try:
print(m[7])
except NotFoundException:
print("Item not found")
print("size of map: " + str(len(m)))
try:
m.remove(5)
print("Item removed")
except NotFoundException:
print("Item not found")
try:
print(m.find(5))
except NotFoundException:
print("Item not found")
print("size of map: " + str(len(m)))
if __name__ == "__main__":
print("\nTESTING BUCKET")
m = Bucket()
test_map(m)
print("\nTESTING HASHMAP")
m = HashMap()
test_map(m)
finally:
chunk.close()
def close(self):
""" cleanup """
for chunk in self.chunks:
self.closeChunk(chunk)
self.chunks = []
if hasattr(self, "m"):
self.m.close()
del self.m
if hasattr(self, "cj"):
del self.cj
if hasattr(self, "info"):
del self.info
if __name__ == "__main__":
url = "http://speedtest.netcologne.de/test_100mb.bin"
from Bucket import Bucket
bucket = Bucket()
bucket.setRate(200 * 1024)
bucket = None
print "starting"
dwnld = HTTPDownload(url, "test_100mb.bin", bucket=bucket)
dwnld.download(chunks=3, resume=True)
class RequestFactory():
def __init__(self, core):
self.lock = Lock()
self.core = core
self.bucket = Bucket()
self.updateBucket()
@property
def iface(self):
return self.core.config["download"]["interface"]
def getRequest(self, pluginName, cj=None):
# TODO: mostly obsolete, browser could be removed completely
req = Browser(self.bucket, self.getOptions())
if cj:
req.setCookieJar(cj)
else:
req.setCookieJar(CookieJar(pluginName))
return req
def getHTTPRequest(self, **kwargs):
""" returns a http request, don't forget to close it ! """
options = self.getOptions()
options.update(kwargs) # submit kwargs as additional options
return HTTPRequest(CookieJar(None), options)
def getURL(self, *args, **kwargs):
""" see HTTPRequest for argument list """
h = HTTPRequest(None, self.getOptions())
try:
rep = h.load(*args, **kwargs)
finally:
h.close()
return rep
def openCookieJar(self, pluginname):
"""Create new CookieJar"""
return CookieJar(pluginname)
def getProxies(self):
""" returns a proxy list for the request classes """
if not self.core.config["proxy"]["proxy"]:
return {}
else:
type = "http"
setting = self.core.config["proxy"]["type"].lower()
if setting == "socks4": type = "socks4"
elif setting == "socks5": type = "socks5"
username = None
if self.core.config["proxy"]["username"] and self.core.config["proxy"]["username"].lower() != "none":
username = self.core.config["proxy"]["username"]
pw = None
if self.core.config["proxy"]["password"] and self.core.config["proxy"]["password"].lower() != "none":
pw = self.core.config["proxy"]["password"]
return {
"type": type,
"address": self.core.config["proxy"]["address"],
"port": self.core.config["proxy"]["port"],
"username": username,
"password": pw,
}
def getOptions(self):
"""returns options needed for pycurl"""
return {"interface": self.iface,
"proxies": self.getProxies(),
"ipv6": self.core.config["download"]["ipv6"]}
def updateBucket(self):
""" set values in the bucket according to settings"""
if not self.core.config["download"]["limit_speed"]:
self.bucket.setRate(-1)
else:
self.bucket.setRate(self.core.config["download"]["max_speed"] * 1024)
