def start(self, m_filter=None):
m_filter = dict() if not m_filter else m_filter
client = MongoClient(self.mongo_host,
self.mongo_port,
maxPoolSize=self.mongo_max_pool_size)
db = client[self.mongo_db_name]
document_name = db[self.mongo_document_name]
mongo_where = m_filter.get('mongo_condition', {})
# get all data from mongoDB db
m_data = document_name.find(mongo_where)
if not self.es:
es = Elasticsearch(
['localhost'],
use_ssl=False,
)
i = 1
spinner = Spinner('Importing... ')
for line in m_data:
docket_content = line
# remove _id from mongo object
del docket_content['_id']
try:
es.index(index=self.es_index_name,
doc_type=self.es_doc_type,
id=i,
body=docket_content)
except Exception as error:
print("Error for ", error)
i += 1
spinner.next()
client.close()
return True
def _perform_spinner_request(self, url, method, method_name, data=None, payload=None, **headers):
if self.state.verbose:
click.echo('Sending {} request to {}'.format(method_name, url))
if payload:
self._echo_spinner_request_payload(payload)
performer = AsyncFileUploader(url, method, data=data, payload=payload, **headers)
performer.start()
spinner = Spinner()
while performer.is_alive():
if self.state.verbose:
spinner.next()
response, error = performer.finish()
# If we have an error and it is an ArcsecondError, raise it.
# As for now, only ArcsecondError could be returned, and there is no
# real point of returning both response and error below. But
# methods in main.py expect them both.
if error and isinstance(error, ArcsecondError):
raise error
if self.state.verbose:
self._echo_spinner_request_result(error, response)
return response, error
def _write_shard(path, source, shard_size, shard_index=None, verbose=True, bar=_DefaultBar):
if shard_index is not None:
path, ext = os.path.splitext(path)
path = path + f"_{shard_index}" + ext
if verbose:
if hasattr(source, "__len__"):
bar = bar(f"Writing to {path}", max=len(source))
else:
bar = Spinner(f"Writing to {path} ")
else:
bar = None
with h5py.File(path, "w") as f:
for example_index, example in enumerate(source):
example = (example,) if isinstance(example, Tensor) else example
for i, tensor in enumerate(example):
key = f"data_{i}"
if key not in f.keys():
f.create_dataset(key, (shard_size, *tensor.shape))
f[key][example_index, ...] = tensor
if bar is not None:
bar.next()
if bar is not None:
bar.finish()
if shard_index is not None:
f.attrs["shard_index"] = int(shard_index)
return path
def get_aws_regions_azs(self):
"""Get regions and AZs"""
ec2 = boto3.client('ec2')
regions_az = {}
if (not (os.path.isfile(REGION_CACHE_FILENAME + self.get_provider()))):
spinner = Spinner(
'\033[1;32;40m getting regions and azs from AWS ')
# Retrieves all regions/endpoints that work with EC2
aws_regions = ec2.describe_regions()
# Get a list of regions and then instantiate a new ec2 client for each
# region in order to get list of AZs for the region
for region in aws_regions['Regions']:
spinner.next()
my_region_name = region['RegionName']
ec2_region = boto3.client('ec2', region_name=my_region_name)
my_region = [{
'Name': 'region-name',
'Values': [my_region_name]
}]
aws_azs = ec2_region.describe_availability_zones(
Filters=my_region)
for az in aws_azs['AvailabilityZones']:
zone = az['ZoneName']
regions_az.setdefault(my_region_name, set()).add(zone)
# cache the results
cache_write_data(REGION_CACHE_FILENAME + self.get_provider(),
regions_az)
else:
regions_az = cache_read_data(REGION_CACHE_FILENAME +
self.get_provider())
return regions_az
def model(name):
path_folder = './' + current() + '/models/'
files = os.listdir(path_folder)
if name + '.py' in files:
ans = input(
'Model with same name already defined. Enter "Y" to replace it or anything else to stop this process: '
).lower()
if ans == 'y' or ans == 'yes':
pass
else:
print('Stopped Execution.')
exit()
path = path_folder + name + '.py'
base = """\
import tensorflow as tf
### Do not duplicate or change the name of the function
### build_model and return the model defined by you.
def build_model():
model = tf.keras.Sequential([
tf.keras.Input(shape=(input_shape)),
...
])
model.compile(loss=<loss>, metrics=<metrics>)
return model
"""
file = open(path, 'w')
file.write(base)
file.close()
spinner = Spinner('Defining a new model with the name: ' + name + ' ')
for _ in range(10):
time.sleep(0.1)
spinner.next()
os.system('kaa ' + path)
def get_data(query, variables):
has_next = True
cursor = None
entities = []
spinner = Spinner('Fetching Github Data')
while has_next:
spinner.next()
variables['cursor'] = cursor
rate_limit = get_rate_limit(client)
handle_rate_limit(rate_limit)
results = json.loads(client.execute(query, variables))
if results['data'] and results['data']['search']['edges']:
nodes = [ edge['node'] for edge in results['data']['search']['edges']]
for node in nodes:
entities.append(parse_data(node))
has_next = results['data']['search']['pageInfo']['hasNextPage']
cursor = results['data']['search']['pageInfo']['endCursor']
else:
logger.warn(f'No data found: {results}')
has_next = False
spinner.finish()
print('\n')
return entities
def project(name):
dirs = os.listdir('./')
if name in dirs:
print(
'Project with same name already present. Start one with another name.'
)
exit()
files = [
'trainers', 'models', 'data', 'data_loader', 'preprocessors',
'results', 'executables'
]
files = ['./' + name] + ['./' + name + '/' + f for f in files]
spinner = Spinner('Generating a New Project: ')
for f in files:
os.system('mkdir ' + f)
time.sleep(0.1)
spinner.next()
time.sleep(0.1)
spinner.next()
print()
print('The project ' + name + ' has been initialised.')
file = open('.active', 'w')
file.write(name)
file.close()
print('Copy paste the data into the ./' + name + '/data/ directory.')
def preprocess(name):
path_folder = './' + current() + '/preprocessors/'
files = os.listdir(path_folder)
if name + '.py' in files:
ans = input(
'Preprocessor with same name already defined. Enter "Y" to replace it or anything else to stop this process: '
).lower()
if ans == 'y' or ans == 'yes':
pass
else:
print('Stopped Execution.')
exit()
path = path_folder + name + '.py'
base = """\
import numpy as np
### Do not duplicate or change the name of the function
### preprocess and return x and y, make sure even if y
### is 1D array, convert it into a 2D one Both should
### ideally be numpy arrays
def preprocess(x, y):
...
return x, y
"""
file = open(path, 'w')
file.write(base)
file.close()
spinner = Spinner(
'Defining a new preprocessing function with the name: ' + name + ' ')
for _ in range(10):
time.sleep(0.1)
spinner.next()
os.system('kaa ' + path)
def chek_resource():
state = ""
spinner = Spinner('Checking for new resource...')
while state != 'FINISHED':
#ps -o pid -C vaas-deploy-vm.py
#pcounter = subprocess.Popen(['ps', '-ef | grep vaas |wc -l'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
pcounter = subprocess.Popen(['pgrep', '[v]aas-deploy-vm'],
stdout=subprocess.PIPE,
shell=False)
nproc = len(pcounter.communicate()[0].split())
if nproc == 1:
state = "FINISHED"
os.popen('setterm -cursor on').read()
else:
#priority queue handle
plist = []
#child = subprocess.Popen(['pgrep', '-f', 'vaas-deploy-vm'], stdout=subprocess.PIPE, shell=False)
father = subprocess.Popen(['pgrep', '[v]aas-deploy-vm'],
stdout=subprocess.PIPE,
shell=False)
response = father.communicate()[0]
for pid in response.split():
plist.append(pid)
# print "io "+ str(os.getpid())
# print "il piu vecchio "+ str(min(plist))
if str(min(plist)) == str(os.getpid()):
state = "FINISHED"
time.sleep(2)
spinner.next()
def start_broadcast(self, broadcast_id):
data = json.dumps({
'_uuid':
self.uuid,
'_uid':
self.username_id,
'should_send_notifications':
int(self.sendNotification),
'_csrftoken':
self.token
})
if self.send_request(endpoint='live/' + str(broadcast_id) + '/start/',
post=self.generate_signature(data)):
print('CTRL+C to quit.')
spinner = Spinner(" - ")
try:
while True:
spinner.next()
except KeyboardInterrupt:
spinner.finish()
pass
except Exception as error:
print(error)
self.end_broadcast(broadcast_id)
def fetch_from_slack(token, channel, oldest):
n_messages = 0
newest = None
oldest = float(oldest)
oldest_ts = str(datetime.fromtimestamp(oldest))
spinner = Spinner('Fetching history for ' +
channel + ' from ' + oldest_ts + ' ')
url = ("https://slack.com/api/conversations.history?token=" + token +
"&channel=" + channel +
"&count=100&inclusive=true&oldest=" + str(round(oldest)))
# records are paged oldest to newest, however the message order
# within a single response is newest to oldest
for message_resp in paged_query(url):
if not message_resp['ok']:
raise ValueError("Error fetching channel history from Slack: ",
message_resp["error"])
messages = message_resp['messages']
newest = messages[0].get('ts', time.time())
n_messages += len(messages)
for message in reversed(messages):
yield message
spinner.next()
print("\nFetched {0} total messages from {1} to {2}".format(
n_messages,
oldest_ts,
str(datetime.fromtimestamp(float(newest)))))
def searchIP():
ip = IP()
count = 0
global ipList
if verbose == "0": #No verbosity
spinner = Spinner('I m Computing...')
while (count < desiredIP):
spinner.next()
try:
r = pyping.ping(ip.generateRandomIP4())
if r.ret_code == 0:
dnsReverse(r.destination)
ipList.insert(count, r.destination)
count = count + 1
except:
print("")
elif verbose == "1": #verbosity
while (count < desiredIP):
ipGenerato = ip.generateRandomIP4()
print "\nGenerated IP4: " + ipGenerato
print "\nTesting it\n"
try:
r = pyping.ping(ipGenerato)
if r.ret_code == 0:
print r.destination + " " + "reachable"
dnsReverse(r.destination)
ipList.insert(count, r.destination)
count = count + 1
elif r.ret_code == 1:
print r.destination + " " + "unreachable"
except:
print("")
return
def get_template_counts(model_id):
import tensorflow as tf
import numpy as np
print('Getting template counts for %s' % model_id)
graph = tf.Graph()
with graph.as_default():
builder = get_builder(model_id)
features, labels = builder.get_inputs(mode='train', repeat=False)
spec = builder.get_estimator_spec(features, labels, mode='eval')
predictions = spec.predictions
probs = predictions['probs']
counts = tf.argmax(probs, axis=-1)
totals = np.zeros((builder.n_templates, ), dtype=np.int32)
saver = tf.train.Saver()
with tf.train.MonitoredSession() as sess:
saver.restore(sess, tf.train.latest_checkpoint(builder.model_dir))
spinner = Spinner()
while not sess.should_stop():
c = sess.run(counts)
for ci in c:
totals[ci] += 1
spinner.next()
# break
spinner.finish()
return totals
def copyTree(src, dst, state, symlinks=False, ignore=None):
errors = []
spinner = Spinner('Loading ')
src_folder_size = get_size_format(get_directory_size(src))
print("___ Folder Size: ", src_folder_size)
while state != 'FINISHED':
for item in os.listdir(src):
if item in ignore_folders and ignore_flag.lower() == "true":
continue
s = os.path.join(src, item)
d = os.path.join(dst, item)
try:
if os.path.isdir(s):
if os.path.isdir(d):
self.recursiveCopyTree(s, d, symlinks, ignore)
else:
shutil.copytree(s, d, symlinks, ignore)
else:
shutil.copy2(s, d)
spinner.next()
except (IOError, os.error) as why:
errors.append((srcname, dstname, str(why)))
# catch the Error from the recursive copytree so that we can
# continue with other files
except Error as err:
errors.extend(err.args[0])
state = "FINISHED"
print("___ Checking Errors: ", errors)
def FindFiles():
load_state = 0
spinner = Spinner('Finding Files ')
while load_state != 'FINISHED':
f = open("logs/path.txt", "a")
cnt = 0
for root, dirs, files in os.walk("/"):
# for root, files in os.walk("/YOUR/TESTING/DIRECTORY"):
for dir in dirs:
if any(s in root for s in EXCLUDE_DIRECTORY):
spinner.next()
pass
else:
for file in files:
if file.endswith(EXCLUDE):
cnt += 1
TARGET = os.path.join(root, file)
f.write(TARGET + '\n')
spinner.next()
print(root)
f.close()
load_state = 'FINISHED'
print()
print("Found {} target files".format(cnt))
print()
def wait_all_replication_to_desire(self,
message="Wait for replica sets: ",
namespace="default"):
"""Check all replicasets and need to be running in the given namespace
:param message: spinner message, defaults to "Wait for replica sets: "
:type message: str, optional
:param namespace: namespace for check, defaults to "default"
:type namespace: str, optional
"""
filter = {"watch": False, "namespace": namespace}
need_wait = True
spinner = Spinner(message)
while (need_wait):
for i in range(15):
spinner.next()
sleep(0.2)
need_wait = False
# Can't use filter because can't get replicasets from specific namespace
api_response = self.get_replica_set()
for rs in api_response.items:
if not self.replica_set_status_check(rs):
need_wait = True
print()
def main(input_path, output_filename):
"""
Google Vison API to CSV
"""
# collect images from path
images = []
spinner = Spinner(('collect images from %s ' % input_path))
for current_file in os.listdir(input_path):
if current_file.endswith('.jpg'):
images.append(current_file)
spinner.next()
spinner.finish()
# analyze images with Google Vision API
responses = []
bar = Bar('analyze images', max=len(images))
for current_img in images:
request = generate_request(input_path + '/' + current_img)
response = call_api(request)
responses.append({'file': current_img, 'response': response})
bar.next()
bar.finish()
#print(json.dumps(responses, indent=4))
create_csv(responses, output_filename)
def nmap(host, user_flags):
flags = ["sudo", "nmap", host]
if user_flags:
flags += user_flags
else:
flags += ["-Pn", "-sS", "-v"]
nmap = subprocess.Popen(flags, stdout=subprocess.PIPE, universal_newlines=True)
ports = []
spinner = Spinner("Nmapping host...")
while True:
spinner.next()
line = nmap.stdout.readline()
if line == "" and nmap.poll() is not None:
break
if line:
if re.search("/.*open", line):
ports.append(line.strip())
# for printing out results
lengths = [29, len(host) + 6, len(sorted(ports, key=len)[-1]) + 9]
longest = sorted(lengths)[-1]
# if port 80 and/or 443 is open and nikto is also called, will nikto those ports
open_ports = []
print("\n\n+{dash}+\n|Host: {host}{space1}|\n+{dash}+\n|PORT{space2}STATE{space3}SERVICE{space4}|\n+{dash}+".format(host=host,space1=" "*(longest-len(host)-4),space2=" "*10,space3=" "*3,space4=" "*(longest-27),dash="-"*(longest+2)))
for p in ports:
items = p.split()
service = " ".join(items[2:])
print("|{port}{space1}{state}{space2}{service}{space3}|".format(port=items[0],space1=" "*(14-len(items[0])),state=items[1],space2=" "*(8-len(items[1])),service=service,space3=" "*(longest-len(service)-20)))
open_ports.append(re.compile(r'/.*').sub('', items[0]))
print("+{dash}+\n".format(dash="-"*(longest+2)))
return open_ports
def run(self):
self.running = True
spinner = Spinner('[' + Fore.GREEN + '*' + St.RESET_ALL + '] ' +
'{} '.format(self.message))
while self.running:
spinner.next()
sleep(0.2)
def crawl(self, pages, depth=1):
spinner = Spinner('Searching ')
for i in range(depth):
newpages = []
for page in pages:
try:
headers = {'User-Agent': 'Mozilla/5.0'}
req = Request(page, None, headers)
c = urlopen(req)
except URLError as e:
print(e)
print("Could not open %s" % page)
continue
soup = BeautifulSoup(c.read(), "html.parser")
links = soup('a')
for link in links:
if 'href' in dict(link.attrs):
url = urljoin(page, link['href'])
url = url.split('#')[0] # remove location portion
values = self.split_words(link.getText())
if len(values) > 3 and self.is_useful(
url) and not self.is_indexed(values):
self.add_to_index(url, values)
# We only parse forum menu pages since they contain thread titles
if self.is_menu(url) and url not in pages:
newpages.append(url)
spinner.next()
# Update pages to crawl
pages = newpages
def _skip(self, frames):
spinner = Spinner(f'Skipping {frames} Frames... ')
frame_number = 0
while frame_number < frames:
spinner.next()
ret, _ = self.cap.read()
if ret: frame_number += 1
logger.info(f'Skipped {frames} frames')
def trainTimeout(dataset):
spiner = Spinner("Training 'no' GAN...")
while (True):
#start = time.time()
for vector_batch in dataset:
train_step(vector_batch)
spiner.next()
spiner.finish()
def start_spin(msg: str):
global spin
spinner = Spinner("%s " % msg)
spin = True
while spin:
spinner.next()
time.sleep(0.1)
spinner.finish()
return
def like_n_swipe(self, amount: int = 1, skip_already_liked: bool = False):
"""
Args:
amount (int): number of posts to like
skip_already_liked (bool): posts already liked count to amount
"""
lk = 0
try:
while lk < amount:
try:
if self.d(resourceId=
"com.instagram.android:id/secondary_label",
text="Sponsored").exists:
self.__skip_sponsored()
if skip_already_liked:
if self.d(
resourceId=
"com.instagram.android:id/row_feed_button_like",
description="Liked").exists:
lk += 1
uiautomator2.logger.info(
"Already liked {}/{}".format(lk, amount))
if self.d(resourceId=
"com.instagram.android:id/row_feed_button_like",
description="Like").exists:
real_liked = len([
self.__click_n_wait(e) for e in self.
d(resourceId=
"com.instagram.android:id/row_feed_button_like",
description="Like")
])
lk = lk + real_liked
self.amount_liked += real_liked
uiautomator2.logger.info("Liking {}/{}".format(
lk, amount))
else:
self.d(scrollable=True).scroll()
except uiautomator2.exceptions.UiObjectNotFoundError as e:
self.__not_found_like(e)
pass
except Exception as e:
self.__treat_exception(e)
return None
if self.amount_liked >= self.amount_to_pause:
uiautomator2.logger.info(
"Total of {} posts liked. Sleeping for {} minutes.".format(
self.amount_liked, self.pause_in_minutes))
i = 0
spinner = Spinner('Waiting ')
while i < (self.pause_in_minutes * 60):
sleep(1)
spinner.next()
i += 1
self.amount_liked = 0
uiautomator2.logger.info("Done: Liked {}/{}".format(lk, amount))
def filtered_file_list_traversal(root_dir,
filtering_type,
extentions_set=None,
remove_empty_items=False,
use_spinner=False):
"""
:param root_dir: str(); r'C:\XComGame\CookedPCConsole'
:param filtering_type: FilteringType()
:param extentions_set: set(); {'.upk', '.txt'}
:return: list() of tuple(); list of (dirpath, dirnames, new_filenames)
"""
if (FilteringType.off != filtering_type) and (extentions_set is None):
raise Exception(
'extentions_set can\'t be None with this filtering type')
spinner = None
spinner_index = 0
if use_spinner:
from progress.spinner import Spinner
spinner = Spinner('Loading ')
result_list = list()
for raw_result in os.walk(root_dir):
dirpath = raw_result[0]
dirnames = raw_result[1]
filenames = raw_result[2]
new_filenames = list()
for filename in filenames:
if FilteringType.off == filtering_type:
new_filenames.append(filename)
elif FilteringType.including == filtering_type:
extention = os.path.splitext(filename)[1]
if extention in extentions_set:
new_filenames.append(filename)
elif FilteringType.excluding == filtering_type:
extention = os.path.splitext(filename)[1]
if extention not in extentions_set:
new_filenames.append(filename)
if remove_empty_items:
if len(new_filenames) > 0:
result = (dirpath, dirnames, new_filenames)
result_list.append(result)
else:
result = (dirpath, dirnames, new_filenames)
result_list.append(result)
if use_spinner:
spinner_index += 1
if spinner_index >= 1000:
spinner_index = 0
spinner.next()
if use_spinner:
print()
return result_list
def wait_and_get_im_sg_fd():
#info("waiting for device to appear")
progressBar = Spinner("Waiting for device: ")
while True:
sg_fd = get_im_sg_fd()
if sg_fd:
progressBar.finish()
return sg_fd
time.sleep(0.5)
progressBar.next()
def umda(literal, domain, iterations, population, bn):
prob_vector_yes = [0.5] * dimension
prob_vector_no = [0.5] * dimension
if domain == 'am':
current_population = generate_initial_population(population)
bar = IncrementalBar('Building probability vectors...', max=iterations)
for i in range(iterations): # Iterations
# Selected Population
interest_programs = get_interest_programs(current_population,
literal, bn)
# Update the probability vector
new_prob_vectors = update_prob_vector(interest_programs[0],
interest_programs[1],
prob_vector_yes,
prob_vector_no)
prob_vector_yes = new_prob_vectors[0]
prob_vector_no = new_prob_vectors[1]
# Generate new population with probability vectors
current_population = generate_population(prob_vector_yes,
prob_vector_no,
population)
bar.next()
bar.finish()
else:
#current_population = generate_population(prob_vector_yes, prob_vector_no, population)
current_population = bn.gen_samples(population)
current_population = [world[0] for world in current_population]
#bar = IncrementalBar('Building probability vectors...', max=iterations)
spinner = Spinner("Processing")
for i in range(iterations): # Iterations
# Selected Population
interest_worlds = get_interest_worlds(current_population, literal,
bn)
# Update the probability vector
new_prob_vectors = update_prob_vector(interest_worlds[0],
interest_worlds[1],
prob_vector_yes,
prob_vector_no)
prob_vector_yes = new_prob_vectors[0]
prob_vector_no = new_prob_vectors[1]
# Generate new population with probability vectors
current_population = generate_population(prob_vector_yes,
prob_vector_no,
population)
spinner.next()
# print("Prob vector for Yes: ", prob_vector_yes)
# print("Prob vector for No: ", prob_vector_no)
#print("Incorrect programs: ", inconsistent_program)
#print("Repeated programs: ", repeated_program)
return [prob_vector_yes, prob_vector_no]
def img_download(request, download_path):
soup = BeautifulSoup(request.text, 'html.parser')
new_src_to_img_list = []
spinner = Spinner('Loading images')
state = 'go'
while state != 'FINISHED':
for link in soup.find_all('img'):
logger.debug('check for having "src" attribute in tag <img>')
if link.get('src') and not urlparse(link.get('src')).scheme:
try:
response = requests.get(
urljoin(
request.url,
link.get("src")
)
)
response.raise_for_status()
except requests.HTTPError as r:
print(
f' Problem with response recieving from tag: '
f'{link.get("src")}. '
f'Error message: {r}')
filename_from_img_link = get_filename_from_tag(
request.url,
link.get('src')
)
new_src_to_img_list.append(filename_from_img_link)
if os.path.isfile(
os.path.join(
download_path,
filename_from_img_link
)
):
raise FileExistsError(
f'File {filename_from_img_link} already exists'
)
with open(
os.path.join(
download_path,
filename_from_img_link
),
"wb"
) as r:
logger.debug(f'downloading image "{link}"')
logger.debug(
'may occur error if dir for download already exist'
)
r.write(response.content)
spinner.next()
state = "FINISHED"
return new_src_to_img_list
def test_bar():
from progress.spinner import Spinner
Spinner.phases = [
'🕐', '🕑', '🕒', '🕓', '🕔', '🕕', '🕖', '🕗', '🕘', '🕙', '🕚', '🕛'
]
_status_bar = Spinner("Downloading.. ", end="www")
for i in range(100):
time.sleep(0.02)
_status_bar.next()
_status_bar.message
_status_bar.finish()
def script_download(request, download_path):
soup = BeautifulSoup(request.text, 'html.parser')
new_src_to_script_list = []
spinner = Spinner('Loading scripts')
state = 'go'
while state != 'FINISHED':
for script in soup.find_all('script'):
if script.get("src"):
file_name = get_filename_from_tag(
request.url,
script.get('src')
)
if not urlparse(script.get('src')).scheme \
or urlparse(script.get('src')).scheme \
and urlparse(script.get('src')).netloc \
== urlparse(request.url).netloc:
try:
response = requests.get(
urljoin(
request.url,
script.get("src")
)
)
response.raise_for_status()
except requests.HTTPError as r:
print(
f' Problem with response recieving from tag: '
f'{script.get("src")}. '
f'Error message: {r}'
)
new_src_to_script_list.append(file_name)
if os.path.isfile(os.path.join(download_path, file_name)):
raise FileExistsError(
f'File {file_name} already exists'
)
with open(
os.path.join(
download_path,
file_name
),
"w"
) as r:
logger.debug(f'downloading script "{script}"')
r.write(response.text)
spinner.next()
state = "FINISHED"
return new_src_to_script_list
def fetch_articles(self, output_progress=False):
spinner = None
if output_progress:
spinner = Spinner('Loading articles ')
articles_index = []
last_fetch = self._configs.get('last_fetch')
offset = 0
count = 20
while(True):
try:
articles = self._pocket.retrieve(
state='unread',
count=count,
offset=offset,
since=last_fetch
)
except PocketException as e:
spinner.finish()
raise_from(self._check_exception(e), e)
if not articles['list']:
break
articles_index.extend(self._get_articles_index(articles))
offset += count
if spinner:
spinner.next()
if spinner:
spinner.finish()
sort_field = self._configs.get('sort_field')
if not sort_field:
sort_field = 'reading_time'
articles_index = sorted(articles_index,
key=itemgetter(sort_field))
self._storage.write(articles_index)
self._configs.set('last_fetch', self._get_timestamp(datetime.now()))
self._configs.write()
def extract_cdr_data(f):
spinner = Spinner('Loading %s:' % f)
with open(f) as fd:
while 1:
l = fd.readline()
spinner.next()
if not l:
raise StopIteration
if l == '':
continue
t = l.split(';')
try:
d = datetime.datetime.strptime(':'.join([t[3], t[4]]),
'%Y%m%d:%H%M%S')
except:
continue
else:
yield {
'cdrType': t[0],
'callingPartyNumberKey': t[1],
'callingSubscriberImsi': t[2],
'dateTime': d,
'chargeableDuration': int(t[5]),
'exchangeIdentity': t[6],
'outgoingRoute': t[7],
'incomingRoute': t[8],
'cellId1stCellCalling': t[9],
'gsmTeleServiceCode': t[10],
'cellIdLastCellCalling': t[11],
'disconnectingParty': t[12],
'callingSubscriberImei': t[13],
'tac': t[14],
'residentCustomerFlag': t[15],
'paymentType': t[16],
'contractStatus': t[17],
'contractStartingDate': t[18],
'country': t[19],
'cityPostalCode': t[20],
'city': t[21]
}
)
)
data = {"token": _token, "ts_to": date}
response = requests.post(files_list_url, data=data)
response = response.json()
if("files" in response.keys()):
fileCount = len(response["files"])
else:
print("The following error message was received from SLACK API: " + str(response['error']))
continue
spinner = Spinner('Looking for files...\n')
if(fileCount > 0):
print("Parsing " + str(fileCount) + " files...\n", end='', file=sys.stdout)
spinner.next()
else:
print("No files older than 10 days found.", file=sys.stdout)
spinning = True
while spinning:
if len(response["files"]) == 0:
spinner.finish()
spinning = False
break
elif(whileCount >= fileCount and whileCount > 1):
spinner.finish()
spinning = False
sys.stdout.flush()
sys.stdout.write("We couldn't delete some files posted by other users on private conversations.")
