def registerToken(requester, service, data, androidID = None):
uuid = shortuuid.uuid()
database = connection.safeguard
userData = database.userData
#delete old token
#print data
#print 'service', service
#print 'requester', requester
#print "data", "{'id':" ,data["id"] ,"}}"
query = {'service': service, 'requester': requester,"data.id": data["id"], "data.androidid" : androidID}
userData.remove(query)
while list(userData.find({'token': uuid}).limit(1)):
uuid = shortuuid.uuid()
if androidID:
data["androidid"] = androidID
userData.insert({"token":uuid, "service" : service, 'requester': requester,"data" : data})
return uuid
def run(**kwargs):
parser = OptionParser(usage="Usage: %prog [options]")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="make lots of noise")
parser.add_option("--logging", dest="logging", default='info', help="Logging level")
parser.add_option("-H", "--host", dest="host", default='localhost', help="RabbitMQ host")
parser.add_option("-P", "--port", dest="port", default=5672, type=int, help="RabbitMQ port")
(options, args) = parser.parse_args()
logging.basicConfig(
format=u'[%(asctime)s] %(filename)s:%(lineno)d %(levelname)-6s %(message)s',
level=getattr(logging, options.logging.upper(), logging.INFO)
)
Listener(
port=options.port,
host=options.host,
handlers=context.handlers,
context=Context(
options=options,
node_uuid=uuid(getfqdn()),
uuid=uuid(),
**kwargs
)
).loop()
def create_slug(self, model_instance, add):
# get fields to populate from and slug field to set
if not isinstance(self._populate_from, (list, tuple)):
self._populate_from = (self._populate_from, )
slug_field = model_instance._meta.get_field(self.attname)
if add or self.overwrite:
# slugify the original field content and set next step to 2
slug_for_field = lambda field: self.slugify_func(getattr(model_instance, field))
slug = self.separator.join(map(slug_for_field, self._populate_from))
#next = 2
next = shortuuid.uuid() # universal unique
next = next[:7] # not universal, but probability of collision is still very low
else:
# get slug from the current model instance
slug = getattr(model_instance, self.attname)
# model_instance is being modified, and overwrite is False,
# so instead of doing anything, just return the current slug
return slug
# strip slug depending on max_length attribute of the slug field
# and clean-up
slug_len = slug_field.max_length
if slug_len:
slug = slug[:slug_len]
slug = self._slug_strip(slug)
original_slug = slug
if self.allow_duplicates:
return slug
# exclude the current model instance from the queryset used in finding
# the next valid slug
queryset = self.get_queryset(model_instance.__class__, slug_field)
if model_instance.pk:
queryset = queryset.exclude(pk=model_instance.pk)
# form a kwarg dict used to impliment any unique_together contraints
kwargs = {}
for params in model_instance._meta.unique_together:
if self.attname in params:
for param in params:
kwargs[param] = getattr(model_instance, param, None)
kwargs[self.attname] = slug
# increases the number while searching for the next valid slug
# depending on the given slug, clean-up
while not slug or queryset.filter(**kwargs):
slug = original_slug
end = '%s%s' % (self.separator, next)
end_len = len(end)
if slug_len and len(slug) + end_len > slug_len:
slug = slug[:slug_len - end_len]
slug = self._slug_strip(slug)
slug = '%s%s' % (slug, end)
kwargs[self.attname] = slug
next = shortuuid.uuid() # universal unique
next = next[:7] # not universal, but probability of collision is still very low
return slug
def generate_unique_slug(instance=None):
""" Generate the unique slug for a model instance """
if instance is not None:
pk = instance.pk if hasattr(instance, 'pk') and type(instance.pk) is not None else random.random()
hash_val = '%s-%s-%s' % (instance.__class__.__name__, pk, datetime.datetime.utcnow())
return shortuuid.uuid(name=hash_val)
else:
return shortuuid.uuid()
def forwards_func(apps, schema_editor):
ClowderUser = apps.get_model("clowder_account", "ClowderUser")
db_alias = schema_editor.connection.alias
users = ClowderUser.objects.all()
for user in users:
user.public_key = shortuuid.uuid()
user.secret_key = shortuuid.uuid()
user.save()
def generate_data(key):
return {
"model": "bookmark.%s" % key,
"pk": shortuuid.uuid(),
"fields": {
"%s" % key: shortuuid.uuid()
}
}
def generate_data(key):
"""generate data."""
return {
"model": "nenga.%s" % key,
"pk": shortuuid.uuid(),
"fields": {
"%s" % key: shortuuid.uuid()
}
}
def setUp(self):
self.required_fields = getattr(User, 'REQUIRED_FIELDS', [])
self.required_fields.append(
getattr(User, 'USERNAME_FIELD', 'username'))
if 'username' in self.required_fields:
self.inviter = User.objects.create(username=uuid())
self.existing = User.objects.create(
email='*****@*****.**', username=uuid())
else:
self.inviter = User.objects.create()
self.existing = User.objects.create(email='*****@*****.**')
def crawling_history(feed_id):
return {
"model": "bookmark.crawling_history",
"pk": shortuuid.uuid(),
"fields": {
"id": shortuuid.uuid(),
"feed": feed_id,
"update_datetime": datetime.now().strftime(
"%Y-%m-%dT%H:%M:%S+00:00")
}
}
def feed_subscription(owner_id, category_id):
return {
"model": "bookmark.feed_subscription",
"pk": shortuuid.uuid(),
"fields": {
"url": "http://%s.example.org/%s" % (shortuuid.uuid()[:4],
shortuuid.uuid()),
"name": shortuuid.uuid(),
"owner": owner_id,
"default_category": category_id
}
}
def main():
parser = OptionParser()
parser.add_option("-s", "--service-registry",
dest="registry_nodes",
default=os.getenv('GILLIAM_SERVICE_REGISTRY', ''),
help="service registry nodes", metavar="HOSTS")
parser.add_option('--name', dest="name")
parser.add_option("-p", "--port", dest="port", type=int,
help="listen port", metavar="PORT", default=9000)
parser.add_option('--host', dest="host", default=None,
help="public hostname", metavar="HOST")
(options, args) = parser.parse_args()
assert options.host, "must specify host with --host"
# logging
format = '%(levelname)-8s %(name)s: %(message)s'
logging.basicConfig(level=logging.INFO, format=format)
formation = os.getenv('GILLIAM_FORMATION', 'executor')
service = os.getenv('GILLIAM_SERVICE', 'api')
instance = options.name or shortuuid.uuid()
clock = Clock()
base_url = os.getenv('DOCKER')
docker = DockerClient(base_url) if base_url else DockerClient()
service_registry_cluster_nodes = options.registry_nodes.split(',')
service_registry = ServiceRegistryClient(
clock, service_registry_cluster_nodes)
cont_runtime = partial(PlatformRuntime, service_registry, options.registry_nodes)
cont_store = ContainerStore(partial(Container, docker, cont_runtime,
service_registry, options.host))
# set-up runtime and store for the one-off containers:
proc_runtime = partial(PlatformRuntime, service_registry, options.registry_nodes,
attach=True)
proc_factory = lambda image, command, env, ports, opts, formation, **kw: Container(
docker, proc_runtime, None, None, image, command, env, ports, opts,
formation, None, shortuuid.uuid(), restart=False, **kw)
proc_store = ContainerStore(proc_factory)
register = partial(service_registry.register, formation, service, instance)
announcement = service_registry.build_announcement(
formation, service, instance, ports={options.port: str(options.port)},
host=options.host)
app = App(clock, cont_store, proc_store, docker, register, announcement)
app.start()
pywsgi.WSGIServer(('', options.port), app.create_api(),
handler_class=WebSocketHandler).serve_forever()
def create_uuid(self):
if not self.version or self.version == 4:
return shortuuid.uuid()
elif self.version == 1:
return shortuuid.uuid()
elif self.version == 2:
raise UUIDVersionError("UUID version 2 is not supported.")
elif self.version == 3:
raise UUIDVersionError("UUID version 3 is not supported.")
elif self.version == 5:
return shortuuid.uuid(name=self.namespace)
else:
raise UUIDVersionError("UUID version %s is not valid." % self.version)
def generate_bookmark(owner_id, category_id, is_hide):
return {
"model": "bookmark.bookmark",
"pk": shortuuid.uuid(),
"fields": {
"url": "http://%s.example.org/%s" % (shortuuid.uuid()[:4],
shortuuid.uuid()),
"title": shortuuid.uuid(),
"category": category_id,
"description": shortuuid.uuid(),
"owner": owner_id,
"is_hide": is_hide
}
}
def pre_save(self, model_instance, add):
"""
This is used to ensure that we auto-set values if required.
See CharField.pre_save
"""
value = super(ShortUUIDField, self).pre_save(model_instance, add)
if self.auto and not value:
# Assign a new value for this attribute if required.
if sys.version_info < (3, 0):
value = unicode(shortuuid.uuid())
else:
value = str(shortuuid.uuid())
setattr(model_instance, self.attname, value)
return value
def ics(request, team_id=None, team_name=None):
if team_id:
this_team = Team.objects.get(id=team_id)
elif team_name:
this_team = Team.objects.get(name=team_name)
home_games = Game.objects.filter(home_team=this_team)
away_games = Game.objects.filter(away_team=this_team)
games = home_games | away_games
games = games.order_by("time", "field")
cal = Calendar()
cal.add('prodid', '-//Breakway Schedules//Soccer Calendars//EN')
cal.add('version', '2.0')
cal.add('X-WR-CALNAME', this_team.name)
cal.add('X-WR-TIMEZONE', 'CST6CDT')
cal.add('X-WR-CALDESC', 'Breakaway Team Schedule')
now_dt = datetime.now()
now_string = "%04d%02d%02dT%02d%02d%02d" % (
now_dt.year,
now_dt.month,
now_dt.day,
now_dt.hour,
now_dt.minute,
now_dt.second
)
for game in games:
event = Event()
try:
summary = '%s vs. %s' % (game.home_team, game.away_team)
except Exception:
summary = 'Breakaway game'
if game.color_conflict:
desc = 'Color conflict! (%s vs. %s)' % (game.away_team.color, game.home_team.color)
summary += ' (color conflict)'
event.add('description', desc)
event.add('summary', summary)
event.add('dtstart', game.time)
event.add('dtend', game.time + timedelta(hours=1))
event.add('dtstamp', datetime.now())
event.add('location', "BreakAway Field %s" % game.field)
event['uid'] = '%s/%[email protected]' % (now_string, shortuuid.uuid())
event.add('priority', 5)
alarm = Alarm()
alarm.add("TRIGGER;RELATED=START", "-PT{0}M".format('45'))
alarm.add('action', 'display')
alarm.add('description', 'Breakaway game')
event.add_component(alarm)
cal.add_component(event)
return HttpResponse(cal.to_ical(), content_type='text/calendar')
def deserialize_person(person, data):
person.uuid = data['uuid']
person.first_name = data['first_name']
person.last_name = data['last_name']
setattr_from(person, 'pref_email', data, parse=get_contact)
person.username = shortuuid.uuid()
def downloadAndSaveImages(url_list, socketid):
try:
uuid = shortuuid.uuid()
directory = os.path.join(conf.PIC_DIR, str(uuid))
if not os.path.exists(directory):
os.mkdir(directory)
for url in url_list[""]:
try:
log_to_terminal(str(url), socketid)
file = requests.get(url)
file_full_name_raw = basename(urlparse(url).path)
file_name_raw, file_extension = os.path.splitext(file_full_name_raw)
# First parameter is the replacement, second parameter is your input string
file_name = re.sub('[^a-zA-Z0-9]+', '', file_name_raw)
f = open(os.path.join(conf.PIC_DIR, str(uuid) + file_name + file_extension), 'wb')
f.write(file.content)
f.close()
imgFile = Image.open(os.path.join(conf.PIC_DIR, str(uuid) + file_name + file_extension))
size = (500, 500)
imgFile.thumbnail(size, Image.ANTIALIAS)
imgFile.save(os.path.join(conf.PIC_DIR, str(uuid), file_name + file_extension))
log_to_terminal('Saved Image: ' + str(url), socketid)
except Exception as e:
print str(e)
return uuid, directory
except:
print 'Exception' + str(traceback.format_exc())
def short_token():
"""
Generate a hash that can be used as an application identifier
"""
hash = hashlib.sha1(shortuuid.uuid())
hash.update(settings.SECRET_KEY)
return hash.hexdigest()[::2]
def wrapExpectationsLocal(cmd):
script = createScript(cmd)
remoteScript = '/tmp/fexpect_'+shortuuid.uuid()
with open(remoteScript, 'w') as filehandle:
filehandle.write(script)
wrappedCmd = 'python '+remoteScript
return wrappedCmd
def save_object(collection, obj):
"""Save an object ``obj`` to the given ``collection``.
``obj.id`` must be unique across all other existing objects in
the given collection. If ``id`` is not present in the object, a
*UUID* is assigned as the object's ``id``.
Indexes already defined on the ``collection`` are updated after
the object is saved.
Returns the object.
"""
if 'id' not in obj:
obj.id = uuid()
id = obj.id
path = object_path(collection, id)
temp_path = '%s.temp' % path
with open(temp_path, 'w') as f:
data = _serialize(obj)
f.write(data)
shutil.move(temp_path, path)
if id in _db[collection].cache:
_db[collection].cache[id] = obj
_update_indexes_for_mutated_object(collection, obj)
return obj
def slug(self, **kwargs):
name = str(self.project.pk) + '-' + str(self.company.pk)
if len(kwargs.keys()) > 0:
name = '{name}{extra}'.format(name=name, extra='-'.join([i for i in kwargs.values()]).encode('utf-8').strip())
return shortuuid.uuid(name=name)
def safe_filename(self, filename):
""" If the file already exists the file will be renamed to contain a
short url safe UUID. This will avoid overwtites.
Arguments
---------
filename : str
A filename to check if it exists
Returns
-------
str
A safe filenaem to use when writting the file
"""
while self.exists(filename):
dir_name, file_name = os.path.split(filename)
file_root, file_ext = os.path.splitext(file_name)
uuid = shortuuid.uuid()
filename = secure_filename('{0}_{1}{2}'.format(
file_root,
uuid,
file_ext))
return filename
def __init__(self, docker, runtime, registry, host,
image, command, env, ports, options,
formation, service, instance,
restart=True, tty=False):
self.docker = docker
self.runtime = runtime
self.registry = registry
self.host = host
self.id = shortuuid.uuid()
cmd = ' '.join(command) if isinstance(command, list) else command
self.log = logging.getLogger('container[{0}/{1}.{2} (image={3}, command="{4}")]'.format(
formation, service, instance, image, cmd))
self.image = image
self.command = command
self.env = env
self.ports = ports
self.options = options
self.formation = formation
self.service = service
self.instance = instance
self.state = 'init'
self.tty = tty
self.reason = None
self.status_code = None
self._stopped = Event()
self._cont_id = None
self._registration = None
self._runtime = None
self._restart = restart
self._reset()
def clean(self):
if self.begins_at > self.ends_at:
self.raise_invalid_bounds('begins_at')
if not self.slug and not self.is_private:
self.slug = slugify(self.title)
elif not self.slug:
self.slug = shortuuid.uuid()
xmin, ymin, xmax, ymax = settings.NYC_BOUNDS
p = self.location
if not p:
raise ValidationError({'location': [
"Location is required"
]})
elif p.x < xmin or p.x > xmax or p.y < ymin or p.y > ymax:
raise ValidationError({'location': [
"Please choose a location in New York City"
]})
rsvp_count = self.eventregistration_set.count()
if self.max_attendees < rsvp_count:
raise ValidationError({'max_attendees': [
"Max attendees cannot be set to a value less than the number "
"of people currently registered (%d)" % rsvp_count]})
def getConguration():
with open(db) as f:
config["database"] = json.load(f)
try:
con = MySQLdb.connect(
config.get("database").get("host"),
config.get("database").get("user"),
config.get("database").get("passwd"),
config.get("database").get("db"),
cursorclass=MySQLdb.cursors.DictCursor,
)
if db:
cur = con.cursor()
cur.execute(
"SELECT * FROM `configurations` WHERE id = (SELECT active FROM `states` ORDER BY id DESC limit 1)"
)
config["fingerprint"] = cur.fetchone()
config["session"] = shortuuid.uuid()
config["remote_ip"] = ipgetter.myip()
config["vpn_ip"] = ni.ifaddresses("tun0")[2][0]["addr"]
print "Connected to " + str(config["vpn_ip"])
config["fingerprint"]["amp_min"] = 10
config["fingerprint"]["plot"] = 0
config["verbose"] = False
config["soundcard"] = {"chunksize": 8096, "channels": 1}
return config
else:
print "Connection unsuccessful"
except MySQLdb.Error, e:
print "MySQL Error [%d]: %s" % (e.args[0], e.args[1])
print "MySQL Error: %s" % str(e)
def long_token():
"""
Generate a hash that can be used as an application secret
"""
hash = hashlib.sha1(shortuuid.uuid())
hash.update(settings.SECRET_KEY)
return hash.hexdigest()
def __init__(self, docker, runtime, registry, host,
image, command, env, ports, options,
formation, service, instance,
restart=True, tty=False):
self.docker = docker
self.runtime = runtime
self.registry = registry
self.host = host
self.id = shortuuid.uuid()
self.log = logging.getLogger('container:%s' % (self.id,))
self.image = image
self.command = command
self.env = env
self.ports = ports
self.options = options
self.formation = formation
self.service = service
self.instance = instance
self.state = 'init'
self.tty = tty
self.reason = None
self.status_code = None
self._stopped = Event()
self._cont_id = None
self._registration = None
self._runtime = None
self._restart = restart
def generate_app_id(length=5):
"""
Generate a short ID for this model.
"""
id = shortuuid.uuid()[:length]
date = datetime.datetime.now().strftime('%d%I%M%p%y-')
return date + id
def add_client():
default_client_secret = shortuuid.uuid()
name = click.prompt('name')
client_id = click.prompt('client_id')
client_secret = click.prompt('client_secret', default=default_client_secret,
hide_input=True, confirmation_prompt=True)
description = click.prompt('description', default='sample client')
is_confidential = click.prompt('is_confidential', default=False, type=bool)
redirect_uris = click.prompt('redirect_uri', )
scopes = click.prompt('scopes', )
client = Client(
client_id=client_id,
client_secret=client_secret,
name=name,
description=description,
is_confidential=is_confidential,
redirect_uris_text=redirect_uris,
default_scopes_text=scopes,
)
db.session.add(client)
try:
db.session.commit()
except IntegrityError as e:
click.echo('Error occurred while add client: <%s>' % client_id)
db.session.rollback()
finally:
print client
def parse_plaintext(body):
"""Parse plaintext and return references in a BibJSON-like dict."""
tmpin = os.path.join("/tmp", shortuuid.uuid())
#tmpin = shortuuid.uuid()
body = unidecode(body)
#print body
preprocessed_body = preprocess_body(body)
#preprocessed_body = "References\n\n" + group_citations(preprocessed_body)
#print preprocessed_body
file = open(tmpin, "w")
file.write(codecs.BOM_UTF8)
file.write(preprocessed_body.encode("utf-8"))
file.close()
try:
parsing = envoy.run("perl ParsCit/bin/citeExtract.pl -m extract_citations %s" % tmpin)
bibjson_citations = xml_to_bibjson(parsing.std_out)
except IOError:
return {"status": "error", "message": "Could not run parser on file"}
try:
os.remove(tmpin)
os.remove(tmpin + ".cite")
os.remove(tmpin + ".body")
except OSError:
pass
return bibjson_citations
def uuid_code():
"""
Returns a short uuid code
"""
return shortuuid.uuid()
def get_a_random_api_token(length=8):
token = shortuuid.uuid()[:length]
return token
'''
Author : ming
date : 2018年1月12日13:43:27
role : 定制 Application
'''
from shortuuid import uuid
from tornado import httpserver, ioloop
from tornado import options as tnd_options
from tornado.options import options, define
from tornado.web import Application as tornado_app
from libs.tornado_libs.web_logs import Logger, ins_log
from ops.settings import IP, PORT
define("port", default=PORT, help="run on the given port", type=int)
define("host", default=IP, help="run port on given host", type=str)
define("progid", default=str(uuid()), help="tornado progress id", type=str)
class Application(tornado_app):
""" 定制 Tornado Application 集成日志 功能 """
def __init__(self,
handlers=None,
default_host="",
transforms=None,
**settings):
tnd_options.parse_command_line() #打印infor日志
Logger(options.progid)
super(Application, self).__init__(handlers, default_host, transforms,
**settings)
http_server = httpserver.HTTPServer(self)
http_server.listen(options.port, address=options.host)
#!/usr/bin/env python
import logging
from shortuuid import uuid
from optparse import OptionParser
from socket import getfqdn
from .listener import Listener
from .context import Context, context
NODE_UUID = uuid(getfqdn())
UUID = uuid()
def run(**kwargs):
parser = OptionParser(usage="Usage: %prog [options]")
parser.add_option("-v",
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="make lots of noise")
parser.add_option("--logging",
dest="logging",
default='info',
help="Logging level")
parser.add_option("-H",
"--host",
dest="host",
default='localhost',
help="RabbitMQ host")
parser.add_option("-P",
"--port",
def add_signal(self,
signal=None,
mdata=None,
group='',
name=None,
compress=False):
"""Add a signal to the file.
Parameters
----------
signal : array
Signal to add.
mdata : dict, optional
Signal metadata.
group : str, optional
Destination signal group.
name : str, optional
Name of the dataset to create.
compress : bool, optional
If True, the signal will be compressed with gzip.
Returns
-------
group : str
Destination group.
name : str
Name of the created signal dataset.
"""
# check inputs
if signal is None:
raise TypeError("Please specify an input signal.")
if mdata is None:
mdata = {}
if name is None:
name = shortuuid.uuid()
# navigate to group
weg = self._join_group(self._signals.name, group)
try:
node = self._file[weg]
except KeyError:
# create group
node = self._file.create_group(weg)
# create dataset
if compress:
dset = node.create_dataset(name, data=signal, compression='gzip')
else:
dset = node.create_dataset(name, data=signal)
# add metadata
dset.attrs['json'] = json.dumps(mdata)
# output
grp = weg.replace('/signals', '')
return utils.ReturnTuple((grp, name), ('group', 'name'))
def facebook_auth():
resp = facebook.authorized_response()
if resp is None or isinstance(resp, OAuthException):
flash(gettext('An error occurred.'), 'danger')
return redirect('/signup')
access_token = resp['access_token']
session['oauth_token'] = (access_token, '')
facebook_me = facebook.get(
'/me?fields=id,name,first_name,last_name,age_range,picture,link,gender,locale,timezone,updated_time,verified'
)
facebook_friends = facebook.get('/me/friends')
authorize = Authorize.query\
.filter_by(oauth_id=facebook_me.data.get('id'), network='facebook')\
.first()
friends = facebook_friends.data.get('summary')
if friends:
friends = friends['total_count']
if authorize is None:
if current_user.is_authenticated():
user_id = current_user.get_id()
else:
user = User(facebook_me.data.get('name'), None, None)
db.session.add(user)
db.session.commit()
user_id = user.id
user_locale = facebook_me.data.get('locale')
if user_locale:
user.locale = user_locale[:2]
db.session.add(user)
db.session.commit()
authorize = Authorize(facebook_me.data.get('id'),
facebook_me.data.get('name'),
facebook_me.data.get('gender'),
facebook_me.data.get('locale'),
facebook_me.data.get('verified'), friends, None,
'facebook', user_id)
else:
authorize.name = facebook_me.data.get('name')
authorize.gender = facebook_me.data.get('gender')
authorize.locale = facebook_me.data.get('locale')
authorize.verified = facebook_me.data.get('verified')
authorize.friends = friends
db.session.add(authorize)
db.session.commit()
if current_app.config['DOWNLOAD_FACEBOOK_PHOTO']:
fb_picture = facebook_me.data.get('picture')
if fb_picture and authorize.picture != fb_picture['data']['url']:
try:
file_name = shortuuid.uuid() + '.jpg'
file_folder = os.path.join(current_app.root_path,
current_app.config['UPLOAD_FOLDER'])
file_path = os.path.join(file_folder, file_name)
url = 'https://graph.facebook.com/{}/picture?type=large&redirect=true&access_token={}'.format(
facebook_me.data.get('id'), access_token)
urllib.request.urlretrieve(url, file_path)
authorize.picture = fb_picture['data']['url']
db.session.add(authorize)
user = User.query.get(authorize.user_id)
user.picture = file_name
db.session.add(user)
db.session.commit()
except Exception:
pass
if current_user.is_anonymous():
user = User.query.get(authorize.user_id)
login_user(user, remember=True)
return redirect(request.args.get('next') or '/user')
import shortuuid print(shortuuid.uuid()) print(shortuuid.uuid(name="example.com")) print(shortuuid.uuid(name="http://www.google.com")) print(shortuuid.ShortUUID().random(length=50)) print(shortuuid.get_alphabet()) import uuid s = shortuuid.encode(uuid.uuid4()) print(s) su = shortuuid.ShortUUID() print(su.uuid())
def generate(ds_name,
tags_list,
filters,
background_dir=None,
rand_backgrounds=False):
start_time = time.time()
# search for available GPUs to speed up generation time
prop = bpy.context.preferences.addons['cycles'].preferences
prop.get_devices()
prop.compute_device_type = 'CUDA'
for device in prop.devices:
if device.type == 'CUDA':
device.use = True
bpy.context.scene.cycles.device = 'GPU'
for scene in bpy.data.scenes:
scene.cycles.device = 'GPU'
#check if folder exists in render, if not, create folder
try:
os.mkdir("render/" + ds_name)
except Exception:
pass
data_storage_path = os.getcwd() + "/render/" + ds_name
#setting file output stuff
output_node = bpy.data.scenes["Render"].node_tree.nodes["File Output"]
output_node.base_path = data_storage_path
#remove all animation
for scene in bpy.data.scenes:
for obj in scene.objects:
obj.animation_data_clear()
shortuuid.set_alphabet('12345678abcdefghijklmnopqrstwxyz')
#np.random.seed(42)
poses = utils.random_rotations(NUM)
lightings = utils.random_rotations(NUM)
images_list = []
img_names = []
# check if background dir is not None and get list of .exr files in that directory
if background_dir is not None:
for f in os.listdir(background_dir):
if not rand_backgrounds:
# background images for dynamically sized moon should be formatted: image_<distance>.exr
# where distance is a float ie. 150.859, this is used to calculate the correct camera rotation so that the moon is in frame
if f.endswith(".exr") or f.endswith(".png") or f.endswith(
".jpg"):
imgnum = f.split("_")[1]
imgnum = imgnum.split(".")[0] + "." + imgnum.split(".")[1]
img_names.append(imgnum)
images_list.append(f)
else:
if f.endswith(".exr") or f.endswith(".png") or f.endswith(
".jpg"):
img_names.append(f)
images_list.append(f)
images_list = sorted(images_list)
node_tree = bpy.data.scenes["Render"].node_tree
check_nodes(filters, node_tree)
reset_filter_nodes(node_tree)
# set default background incase base blender file is messed up
bpy.data.worlds["World"].node_tree.nodes[
'Environment Texture'].image = bpy.data.images["Moon1.exr"]
for i, (pose, lighting) in enumerate(zip(poses, lightings)):
for scene in bpy.data.scenes:
scene.unit_settings.scale_length = 1 / SCALE
nmi = np.random.uniform(low=0.5, high=6)
distance = nm_to_bu(nmi)
if img_names:
random_name = random.choice(img_names)
if not rand_backgrounds:
# 75/25 split between images with moon background and deep space background
if np.random.uniform(0, 1) < 0.75:
if img_names:
# moon background - uniform distribution over disk slightly larger than moon - hopefully
r = (45 / float(random_name)) * MOON_RADIUS * 2 * np.sqrt(
np.random.random())
t = np.random.uniform(low=0, high=2 * np.pi)
background = Euler([
0, MOON_CENTERX - (r / 2) * np.cos(t),
MOON_CENTERY + (r / 2) * np.sin(t)
])
else:
r = MOON_RADIUS * np.sqrt(np.random.random())
t = np.random.uniform(low=0, high=2 * np.pi)
background = Euler([
0, MOON_CENTERX - r * np.cos(t),
MOON_CENTERY + r * np.sin(t)
])
else:
background = Euler([0, 0, 0])
else:
# no randomized background images should have a [0,0,0] background as gateway will be centered on weird distortion pattern
r = MOON_RADIUS * np.sqrt(np.random.random())
t = np.random.uniform(low=0, high=2 * np.pi)
background = Euler([
0, MOON_CENTERX - r * np.cos(t), MOON_CENTERY + r * np.sin(t)
])
offset = np.random.uniform(low=0.0, high=1.0, size=(2, ))
position = np.random.uniform(low=0.0, high=1.0, size=(3, ))
bpy.context.scene.frame_set(0)
frame = starfish.Frame(position=position,
background=background,
pose=pose,
lighting=lighting,
distance=distance,
offset=offset)
frame.setup(bpy.data.scenes['Real'], bpy.data.objects["Gateway"],
bpy.data.objects["Camera"], bpy.data.objects["Sun"])
# load new Environment Texture
if img_names:
if not rand_backgrounds:
image = bpy.data.images.load(filepath=background_dir +
"/image_" + random_name + ".exr")
else:
image = bpy.data.images.load(filepath=os.path.join(
background_dir, random.choice(images_list)))
bpy.data.worlds["World"].node_tree.nodes[
'Environment Texture'].image = image
set_filter_nodes(filters, node_tree)
#create name for the current image (unique to that image)
name = shortuuid.uuid()
output_node.file_slots[0].path = "image_" + str(name) + "#"
output_node.file_slots[1].path = "mask_" + str(name) + "#"
mask_filepath = os.path.join(output_node.base_path,
"mask_" + str(name) + "0.png")
meta_filepath = os.path.join(output_node.base_path,
"meta_" + str(name) + "0.json")
# render
bpy.ops.render.render(scene="Render")
#Tag the pictures
frame.tags = tags_list
# add metadata to frame
frame.sequence_name = ds_name
# run color normalization with labels plus black background
normalize_mask_colors(mask_filepath,
list(LABEL_MAP.values()) + [(0, 0, 0)])
# get bbox and centroid and add them to metadata
frame.bboxes = get_bounding_boxes_from_mask(mask_filepath, LABEL_MAP)
frame.centroids = get_centroids_from_mask(mask_filepath, LABEL_MAP)
with open(
os.path.join(output_node.base_path,
"meta_" + str(name) + "0.json"), "w") as f:
f.write(frame.dumps())
print("===========================================" + "\r")
time_taken = time.time() - start_time
print("------Time Taken: %s seconds----------" % (time_taken) + "\r")
print("Data stored at: " + data_storage_path)
bpy.ops.wm.quit_blender()
from azure.storage.blob import BlobServiceClient
import boto3
from dotenv import find_dotenv, load_dotenv
from pytest_cases import fixture, fixture_union
from shortuuid import uuid
from cloudpathlib import AzureBlobClient, AzureBlobPath, S3Client, S3Path
import cloudpathlib.azure.azblobclient
import cloudpathlib.s3.s3client
from .mock_clients.mock_azureblob import mocked_client_class_factory
from .mock_clients.mock_s3 import mocked_session_class_factory
load_dotenv(find_dotenv())
SESSION_UUID = uuid()
# ignore these files when uploading test assets
UPLOAD_IGNORE_LIST = [
".DS_Store", # macOS cruft
]
@fixture()
def assets_dir() -> Path:
"""Path to test assets directory."""
return Path(__file__).parent / "assets"
class CloudProviderTestRig:
"""Class that holds together the components needed to test a cloud implementation."""
stmthead = "UPDATE " + tab + " set image_file = '%s' where id = %d"
stmt = "SELECT id, image_url, genus, family FROM " + tab + " where image_url <>'' and image_url is not null and (image_file is null or image_file = '') order by id"
cur.execute(stmt)
i = 0
for row in cur:
i = i + 1
url = row[1]
if app == 'other' and row[3]:
family = row[3].title()
urlparts = re.search('\/(.*)(\?)?', url)
a = urlparts.group(1)
# ext = a.split('.')[-1]
ext = 'jpg'
uid = shortuuid.uuid()
fname = row[2] + '_' + shortuuid.uuid() + "." + ext
# fname = "%s_%09d_%09d.jpg" % (type, row[0], row[2])
if not url or url == '':
print("Bad", url)
cur.nextset()
try:
# html = urlopen(url)
if app == 'other':
imgdir = "/mnt/static/utils/images/" + family + "/"
Path(imgdir).mkdir(parents=True, exist_ok=True)
local_filename, headers = urllib.request.urlretrieve(
url, imgdir + fname)
stmt = stmthead % (fname, row[0])
def add_event(self,
ts=None,
values=None,
mdata=None,
group='',
name=None,
compress=False):
"""Add an event to the file.
Parameters
----------
ts : array
Array of time stamps.
values : array, optional
Array with data for each time stamp.
mdata : dict, optional
Event metadata.
group : str, optional
Destination event group.
name : str, optional
Name of the dataset to create.
compress : bool, optional
If True, the data will be compressed with gzip.
Returns
-------
group : str
Destination group.
name : str
Name of the created event dataset.
"""
# check inputs
if ts is None:
raise TypeError("Please specify an input array of time stamps.")
if values is None:
values = []
if mdata is None:
mdata = {}
if name is None:
name = shortuuid.uuid()
# navigate to group
weg = self._join_group(self._events.name, group)
try:
node = self._file[weg]
except KeyError:
# create group
node = self._file.create_group(weg)
# create new event group
evt_node = node.create_group(name)
# create datasets
if compress:
_ = evt_node.create_dataset('ts', data=ts, compression='gzip')
_ = evt_node.create_dataset('values',
data=values,
compression='gzip')
else:
_ = evt_node.create_dataset('ts', data=ts)
_ = evt_node.create_dataset('values', data=values)
# add metadata
evt_node.attrs['json'] = json.dumps(mdata)
# output
grp = weg.replace('/events', '')
return utils.ReturnTuple((grp, name), ('group', 'name'))
def short_token():
"""Generate a 20-character random token"""
hash = hashlib.sha1(shortuuid.uuid().encode('utf-8'))
hash.update(settings.SECRET_KEY.encode('utf-8'))
return hash.hexdigest()[::2]
async def handler(messages: List[WorkerMsg]):
for msg in messages:
site_id = msg['site_id']
url = msg['url']
domain = msg.get('domain', '')
pattern = "*"
html = await get_page(url)
new_page = True
if msg.get('doc_id'):
new_page = False
key = msg['obj_key']
meta_key = msg['meta_obj_key']
doc_id = msg['doc_id']
else:
key = shortuuid.uuid(name=url)
meta_key = f"{key}.metadata.json"
doc_id = f"{site_id}:{key}"
f = BytesIO(html.raw_html)
BUCKET.upload_fileobj(f, key, ExtraArgs={"ACL": "bucket-owner-full-control"})
doc_title = html.find("title", first=True).text
metadata = {
"DocumentId": doc_id,
"Attributes": {
"_source_uri": url,
"site_id": site_id,
"domain": domain,
},
"Title": doc_title,
"ContentType": "HTML",
}
meta_obj = BytesIO(json.dumps(metadata, ensure_ascii=False).encode('utf-8'))
BUCKET.upload_fileobj(meta_obj, meta_key, ExtraArgs={"ACL": "bucket-owner-full-control"})
now = arrow.utcnow()
updates: list = [
Page.scraped.set(True),
Page.doc_title.set(doc_title),
Page.last_scraped_at.set(now.timestamp)
]
if new_page:
updates += [
Page.doc_id.set(doc_id),
Page.obj_key.set(key),
Page.meta_obj_key.set(meta_key),
]
try:
Page(site_id, url).update(updates)
except Page.DoesNotExist:
print(f"Fail update scrap {url} result")
continue
# get links
links = {refine_url(l) for l in html.absolute_links if l != url and (domain in l) and verify(pattern, url)}
for link in links:
try:
page = Page.get(site_id, link)
page.update(
[Page.scraped.set(False)],
# 한시간 이내 수집 했으면 수집 안함
condition=~Page.last_scraped_at.exists() | Page.last_scraped_at < now.shift(hours=-1).timestamp
)
except Page.DoesNotExist:
page = Page(
site_id, link,
_type='html',
user=msg['user'],
)
page.save()
except Exception:
print(f'already scraped {site_id} : {url}')
def create_short_url():
return shortuuid.uuid()[:7]
def generate_id():
import shortuuid
return shortuuid.uuid()
Universal_pat = re.compile(r"[a-zA-Z\_][0-9a-zA-Z\_]*")
comma_pat = re.compile(r"\s*,\s*")
logger = Logger("sys").getLogger
cli_logger = Logger("cli").getLogger
err_logger = Logger("error").getLogger
plugin_logger = Logger("plugin").getLogger
access_logger = Logger("access").getLogger
md5 = lambda pwd: hashlib.md5(pwd).hexdigest()
hmac_sha256 = lambda message: hmac.new(key="273d32c8d797fa715190c7408ad73811",
msg=message,
digestmod=hashlib.sha256).hexdigest()
gen_token = lambda n=32: b32encode(uuid4().hex)[:n]
gen_requestId = lambda: str(uuid4())
gen_fingerprint = lambda n=16, s=2: ":".join(
["".join(random.sample("0123456789abcdef", s)) for i in range(0, n)])
gen_uniqueId = lambda: shortuuid.uuid()
def ip_check(ip):
if ip and isinstance(ip, (str, unicode)):
return ip_pat.match(ip)
def email_check(email):
if email and isinstance(email, (str, unicode)):
return mail_pat.match(email)
def phone_check(phone):
if phone and isinstance(phone, (str, unicode)):
return mobilephone_pat.match(phone)
def main_pipeline(config):
"""
runs the pipeline subcommand. Runs clustering (or just splitting if clustering is skipped) and sets up
submission scripts for the other pipeline stages/steps
"""
from shortuuid import uuid
batch_id = 'batch-' + str(uuid())
conversion_dir = mkdirp(os.path.join(config.output, 'converted_inputs'))
pairing_inputs = []
submitall = []
jobid_var_index = 0
config.output = os.path.abspath(config.output)
def get_prefix(filename):
return re.sub(r'\.[^\.]+$', '', os.path.basename(filename))
job_name_by_output = {}
for libconf in config.libraries.values():
base = os.path.join(
config.output,
'{}_{}_{}'.format(libconf.library, libconf.disease_status,
libconf.protocol))
log('setting up the directory structure for', libconf.library, 'as',
base)
libconf.inputs = run_conversion(config, libconf, conversion_dir)
# run the cluster stage
cluster_output = mkdirp(os.path.join(
base, SUBCOMMAND.CLUSTER)) # creates the output dir
merge_args = {'batch_id': batch_id, 'output': cluster_output}
merge_args['split_only'] = SUBCOMMAND.CLUSTER in config.skip_stage
merge_args.update(config.reference.items())
merge_args.update(config.cluster.items())
merge_args.update(libconf.items())
log('clustering', '(split only)' if merge_args['split_only'] else '')
inputs = cluster_main.main(log_args=True, **merge_args)
for inputfile in inputs:
prefix = get_prefix(inputfile) # will be batch id + job number
dependency = ''
if SUBCOMMAND.VALIDATE not in config.skip_stage:
outputdir = mkdirp(
os.path.join(base, SUBCOMMAND.VALIDATE, prefix))
command = build_validate_command(config, libconf, inputfile,
outputdir)
# build the submission script
options = {k: config.schedule[k] for k in STD_SUBMIT_OPTIONS}
options['stdout'] = outputdir
options['jobname'] = 'MV_{}_{}'.format(libconf.library, prefix)
if libconf.is_trans():
options[
'memory_limit'] = config.schedule.trans_validation_memory
else:
options['memory_limit'] = config.schedule.validation_memory
script = SubmissionScript(command, config.schedule.scheduler,
**options)
scriptname = script.write(os.path.join(outputdir, 'submit.sh'))
submitall.append('vjob{}=$({} {})'.format(
jobid_var_index,
SCHEDULER_CONFIG[config.schedule.scheduler].submit,
scriptname))
# for setting up subsequent jobs and holds
outputfile = os.path.join(outputdir, VALIDATION_PASS_PATTERN)
job_name_by_output[outputfile] = options['jobname']
inputfile = outputfile
dependency = SCHEDULER_CONFIG[
config.schedule.scheduler].dependency(
'${{vjob{}##* }}'.format(jobid_var_index))
# annotation cannot be skipped
outputdir = mkdirp(os.path.join(base, SUBCOMMAND.ANNOTATE, prefix))
command = build_annotate_command(config, libconf, inputfile,
outputdir)
options = {k: config.schedule[k] for k in STD_SUBMIT_OPTIONS}
options['stdout'] = outputdir
options['jobname'] = 'MA_{}_{}'.format(libconf.library, prefix)
options['memory_limit'] = config.schedule.annotation_memory
script = SubmissionScript(command, config.schedule.scheduler,
**options)
scriptname = script.write(os.path.join(outputdir, 'submit.sh'))
submitall.append('ajob{}=$({} {} {})'.format(
jobid_var_index,
SCHEDULER_CONFIG[config.schedule.scheduler].submit, dependency,
scriptname))
outputfile = os.path.join(outputdir, ANNOTATION_PASS_PATTERN)
pairing_inputs.append(outputfile)
job_name_by_output[outputfile] = options['jobname']
jobid_var_index += 1
# set up scripts for the pairing held on all of the annotation jobs
outputdir = mkdirp(os.path.join(config.output, SUBCOMMAND.PAIR))
args = config.pairing.flatten()
args.update({
'output': outputdir,
'annotations': config.reference.annotations_filename
})
command = ['{} {}'.format(PROGNAME, SUBCOMMAND.PAIR)]
command.extend(stringify_args_to_command(args))
command.append('--inputs {}'.format(' \\\n\t'.join(pairing_inputs)))
command = ' \\\n\t'.join(command)
options = {k: config.schedule[k] for k in STD_SUBMIT_OPTIONS}
options['stdout'] = outputdir
options['jobname'] = 'MP_{}'.format(batch_id)
script = SubmissionScript(command, config.schedule.scheduler, **options)
scriptname = script.write(os.path.join(outputdir, 'submit.sh'))
submitall.append('jobid=$({} {} {})'.format(
SCHEDULER_CONFIG[config.schedule.scheduler].submit,
SCHEDULER_CONFIG[config.schedule.scheduler].dependency(':'.join([
'${{ajob{}##* }}'.format(i) for i in range(0, jobid_var_index)
])), scriptname))
# set up scripts for the summary held on the pairing job
outputdir = mkdirp(os.path.join(config.output, SUBCOMMAND.SUMMARY))
args = dict(output=outputdir,
flanking_call_distance=config.pairing.flanking_call_distance,
split_call_distance=config.pairing.split_call_distance,
contig_call_distance=config.pairing.contig_call_distance,
spanning_call_distance=config.pairing.spanning_call_distance,
dgv_annotation=config.reference.dgv_annotation_filename,
annotations=config.reference.annotations_filename,
inputs=os.path.join(config.output,
'pairing/mavis_paired*.tab'))
args.update(config.summary.items())
command = ['{} {}'.format(PROGNAME, SUBCOMMAND.SUMMARY)]
command.extend(stringify_args_to_command(args))
command = ' \\\n\t'.join(command)
options = {k: config.schedule[k] for k in STD_SUBMIT_OPTIONS}
options['stdout'] = outputdir
options['jobname'] = 'MS_{}'.format(batch_id)
script = SubmissionScript(command, config.schedule.scheduler, **options)
scriptname = script.write(os.path.join(outputdir, 'submit.sh'))
submitall.append('{} {} {}'.format(
SCHEDULER_CONFIG[config.schedule.scheduler].submit,
SCHEDULER_CONFIG[config.schedule.scheduler].dependency('${jobid##* }'),
scriptname))
# now write a script at the top level to submit all
submitallfile = os.path.join(config.output,
'submit_pipeline_{}.sh'.format(batch_id))
log('writing:', submitallfile)
with open(submitallfile, 'w') as fh:
for line in submitall:
fh.write(line + '\n')
async def post(request: Request):
"""
上传app, format[安装包-package: file, 更新说明(默认为空)-msg: str]
:param request:
:return:
"""
file = request.files.get('package')
update_msg = request.form.get('msg') or ''
if not file:
log.warning('not upload file')
raise BadRequest('not find file')
session = Session()
while 1:
# 安装包id
fid = uuid('{}-{}'.format(id(Request), time.time()), 16)
if not session.query(AppVersionModel).filter_by(id=fid).count():
break
file_name = '{}.{}'.format(fid, file.name[file.name.rfind('.') + 1:])
file_path = '{}/{}'.format(Config.app_dir, file_name)
# 保存安装包
async with aiofiles.open(file_path, 'wb+') as f:
await f.write(file.body)
log.debug('save upload success: {}'.format(file_path))
package = await PackageParse.parse(file_path)
if not package:
os.remove(file_path)
raise BadRequest('the file is not support')
app_query = session.query(AppModel).filter_by(
package_name=package.package_name, type=package.app_type)
if app_query.count(): # 已存在
app = app_query.one()
else: # 不存在
# 生成短链
while 1:
short_chain = random(8)
if not session.query(AppModel).filter_by(
short_chain_uri_=short_chain).count():
break
app_uuid = uuid(
'+_{}-{}_+'.format(package.app_type, package.package_name), 16)
icon_name = '{}.{}'.format(
app_uuid, package.icon_path[package.icon_path.rfind('.') + 1:])
app = AppModel(type=package.app_type,
short_chain_uri_=short_chain,
detail='',
name=package.app_name,
package_name=package.package_name,
icon_='{}/{}'.format(Config.icon_dir, icon_name),
icon_uri_='{}/{}'.format(Config.static_icon,
icon_name))
session.add(app)
session.commit()
# 保存图标
await package.save_icon(app.icon_)
file_byte = os.path.getsize(file_path)
file_size = Byte.pretty(file_byte)
app_version = AppVersionModel(
id=fid,
version_name=package.version_name,
version_code=package.version_code,
update_msg=update_msg,
size=file_size,
package_='{}/{}'.format(Config.app_dir, file_name),
package_uri_='{}/{}'.format(Config.static_app, file_name),
app_id=app.id)
session.add(app_version)
session.commit()
return JsonResult.ok().response_json()
try:
connection = psycopg2.connect(host=database_ids.host, database=database_ids.database, user=database_ids.user, password=database_ids.password)
# declare a cursor
cur = connection.cursor()
# requests and reception of the data
# createClient = "INSERT INTO fake_clients_clone VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
refresh = "DELETE FROM fake_clients"
createClient = "INSERT INTO fake_clients VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
# empties the table
cur.execute(refresh)
count = 0
while count < 200 :
client_id = shortuuid.uuid()
name = fake.name()
address = fake.address()
phone_number = fake.phone_number()
birthdate = str(fake.date_of_birth(None, 40, 65))
email = fake.free_email(),
gender = randomStrRange("H", "F")
et = fake.date_between(start_date='-90d', end_date='today') # need the raw version to make the last_maintenance
engie_time = str(et)
nb_panneaux = randomIntRange(1, 6)
last_activity = fake.date_between(start_date='-6d', end_date='today'),
time_activity = scoringActivityApp()
profil = randomIntRange(1, 4) # "Économe", "Autonome", "Écologie"
famille = randomIntRange(1, 9) # "IFaP", "IFoP", "IFaN", "IFoN", "PFaP", "PFoP", "PFaN", "PFoN"
feeling = randomIntRange(3, 6) # "Curieux", "Harmonieux", "Heureux"
satisfaction = randomIntRange(20, 100)
def mkdtemp(base_directory=None):
prefix = "dvc-test.{}.".format(os.getpid())
suffix = ".{}".format(shortuuid.uuid())
return tempfile.mkdtemp(
prefix=prefix, suffix=suffix, dir=base_directory
)
def tmp_fname(fname):
""" Temporary name for a partial download """
return fspath(fname) + "." + uuid() + ".tmp"
def track_trajectory(self):
self._is_tracking = True
id = shortuuid.uuid()
cap = self._cap
# MIL appears to work best...
tracker = self.get_tracker(self._tracker_type)
cap_fps = cap.get(cv2.CAP_PROP_FPS)
current_frame = int(cap.get(
cv2.CAP_PROP_POS_FRAMES)) # int(cap_fps * start_sec)
duration_frames = int(cap_fps * self._duration_sec)
# cap.set(cv2.CAP_PROP_POS_FRAMES, current_frame)
time = (current_frame - 1) * cap_fps
resw = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
resh = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
res = (resw, resh)
frame = self._current_frame
cv2.putText(frame, "SET MARK", (100, 150), cv2.FONT_HERSHEY_SIMPLEX,
0.75, (50, 170, 50), 2)
cv2.imshow(WIN_NAME, frame)
if self._save_video:
fourcc = cv2.VideoWriter_fourcc(*'DIVX')
out_fname = self._root_path + self._filename + \
'.multi_track_{0}.avi'.format(id)
out = cv2.VideoWriter(out_fname, fourcc, cap_fps, res)
# Define an initial bounding box
# bbox = (287, 23, 86, 320)
bbox = cv2.selectROI(WIN_NAME, frame, True)
ok = tracker.init(frame, bbox)
# Uncomment the line below to select a different bounding box
# selector_frame = frame.copy()
csv_fname = self._root_path + self._filename + \
'.{0}.{1}.{2}.{3}.csv'.format(id, bbox[2], bbox[3], current_frame)
csvfile = open(csv_fname, 'w', newline='')
bac_writer = csv.writer(csvfile,
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
bac_writer.writerow(['BacteriaId', 'Frame', 'Time (s)', 'X', 'Y'])
invalid = False
while cap.isOpened() and current_frame < duration_frames:
# ret, frame = cap.read()
frame = self.next_frame()
# Start timer
timer = cv2.getTickCount()
# Update tracker
ok, bbox = tracker.update(frame)
# Calculate Frames per second (FPS)
fps = cv2.getTickFrequency() / (cv2.getTickCount() - timer)
# Draw bounding box
if ok:
p1 = (int(bbox[0]), int(bbox[1]))
p2 = (int(bbox[0] + bbox[2]), int(bbox[1] + bbox[3]))
cv2.rectangle(frame, p1, p2, (255, 0, 0), 2, 1)
bac_writer.writerow([
id, current_frame, "{:.3f}".format(time),
int(bbox[0] + bbox[2] / 2),
int(bbox[1] + bbox[3] / 2)
])
else:
# Tracking failure
cv2.putText(frame, "Tracking failure detected", (100, 80),
cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0, 0, 255), 2)
if self._save_video:
# Display tracker type on frame
# cv2.putText(frame, "2.131 Bacterial Tracker", (100, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50, 170, 50), 2);
# Display FPS on frame
cv2.putText(frame, "FPS : " + str(int(fps)), (100, 50),
cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50, 170, 50), 2)
out.write(frame)
# Display result
cv2.imshow(WIN_NAME, frame)
cv2.setTrackbarPos(POS_TRACKBAR, WIN_NAME,
int(self._cap.get(cv2.CAP_PROP_POS_FRAMES)))
current_frame = current_frame + 1
time = time + 1.0 / cap_fps
# stop recording if ESC or space pressed
k = cv2.waitKey(1) & 0xff
if k == 27 or k == ord(" "):
invalid = True
break
csvfile.close()
if self._save_video:
out.release()
if invalid:
cv2.putText(frame, "CANCELLED", (100, 80),
cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0, 0, 255), 2)
cv2.imshow(WIN_NAME, frame)
os.remove(csv_fname)
os.remove(out_fname)
cv2.imshow(WIN_NAME, frame)
self._current_frame = frame
self._is_tracking = False
self.load_existing_trackers()
def calculate_derived(self):
"""
Derives additional configuration values necessary for training from the current config
"""
# wandb
# sets a unique wandb group
if self.wandb_group is None:
# if none is defined a uuid is set for the run
self.wandb_group = shortuuid.uuid()
# number of gpus
# Get number of GPUs param or hostfile to determine train_batch_size
num_gpus = self.num_gpus
if num_gpus is None:
num_gpus = -1 # set -1 for backwards compatibility to old default value
if num_gpus < 1:
if self.hostfile is not None or os.path.exists(DLTS_HOSTFILE):
hostfile_path = self.hostfile or DLTS_HOSTFILE
resources = obtain_resource_pool(hostfile_path, self.include or "", self.exclude or "")
num_gpus = sum(map(len, resources.values()))
else:
num_gpus = torch.cuda.device_count()
self.update_value("num_gpus", num_gpus)
logging.info(
self.__class__.__name__ + ".calculate_derived() " + f"Total number of GPUs determined to be: {self.num_gpus}")
# get world size in the model/pipe parallel case, the actual `world size` deepspeed uses is the size of the
# data-parallel group, or (num_gpus / mp_size) / pp_size
pp_size = self.pipe_parallel_size
pp_size = pp_size if pp_size >= 1 else 1
mp_size = self.model_parallel_size
mp_size = mp_size if mp_size >= 1 else 1
self.update_value("model_parallel_size", mp_size)
# pp_size and mp_size are only used here to compute dp world size and nowhere else.
dp_world_size = ((num_gpus / pp_size) / mp_size)
if not (dp_world_size % 1 == 0):
error_message = self.__class__.__name__ + ".calculate_derived() " + f"(num_gpus / pp_size) / mp_size [({num_gpus} / {pp_size}) / {mp_size}] must be a whole number"
logging.error(error_message)
raise AssertionError(error_message)
# Automatically derive train_batch_size = train_micro_batch_size_per_gpu*num_gpus*gradient_accumulation_steps
train_batch_size, train_micro_batch_size_per_gpu, gradient_accumulation_steps = self.calculate_batch_parameters(
dp_world_size=dp_world_size,
train_batch=self.train_batch_size,
micro_batch=self.train_micro_batch_size_per_gpu,
grad_acc=self.gradient_accumulation_steps
)
self.check_batch_parameters(
dp_world_size=dp_world_size,
train_batch=train_batch_size,
micro_batch=train_micro_batch_size_per_gpu,
grad_acc=gradient_accumulation_steps
)
self.update_values({
# batch size params
"train_batch_size": train_batch_size,
"train_micro_batch_size_per_gpu": train_micro_batch_size_per_gpu,
"gradient_accumulation_steps": gradient_accumulation_steps,
"batch_size": train_micro_batch_size_per_gpu,
# duplicate items
"gas": self.gradient_accumulation_steps,
"clip_grad": self.gradient_clipping,
})
# derive precision
if (self.fp16 or {}).get("type", self.precision) == "bfloat16":
self.update_value("precision", "bfloat16")
elif (self.fp16 or {}).get("enabled", False):
self.update_value("precision", "fp16")
else:
self.update_value("precision", "fp32")
# zero optimization
if self.zero_optimization is None:
self.zero_optimization = copy.deepcopy(ZERO_DEFAULTS) # a dict is overwritten and not updated key by key
self.update_values({
"zero_stage": self.zero_optimization.get('stage', ZERO_DEFAULTS['stage']),
"zero_reduce_scatter": self.zero_optimization.get('reduce_scatter', ZERO_DEFAULTS['reduce_scatter']),
"zero_contiguous_gradients": self.zero_optimization.get('contiguous_gradients',
ZERO_DEFAULTS['contiguous_gradients']),
"zero_reduce_bucket_size": self.zero_optimization.get('reduce_bucket_size',
ZERO_DEFAULTS['reduce_bucket_size']),
"zero_allgather_bucket_size": self.zero_optimization.get('allgather_bucket_size',
ZERO_DEFAULTS['allgather_bucket_size'])
})
# optimizer and scheduler
opt_params = self.optimizer or {"type": OPT_DEFAULT, "params": OPT_PARAMS_DEFAULTS}
self.update_values({
"optimizer_type": opt_params.get('type', OPT_DEFAULT),
"lr": opt_params['params'].get('lr', OPT_PARAMS_DEFAULTS['lr'])
})
if self.optimizer_type.lower() == "onebitadam":
# onebitadam needs to instantiated by deepspeed, and so we need to pass deepspeed scheduler args
# for all other optimizers, the scheduling is handled by megatron
self.scheduler = {
"type": "WarmupDecayLR", # for now this is the only ds scheduler offering decay
"params": {
"warmup_min_lr": 0,
"warmup_max_lr": self.lr,
"warmup_num_steps": int(self.train_iters * self.warmup),
"total_num_steps": self.lr_decay_iters or self.train_iters
}}
# Fp16 loss scaling.
self.update_value("dynamic_loss_scale", self.loss_scale is None)
# Update 'is pipe parallel' flag
# if we set pipe_parallel_size to 0 or 1, GPT2ModelPipe.to_sequential() is called, and we run training with
# the sequential model without the PipelineModule wrapper to avoid the overhead it incurs
self.update_value("is_pipe_parallel", self.pipe_parallel_size >= 1)
# Attention config
if self.attention_config is None:
self.update_value("attention_config", [[["global"], self.num_layers]])
self.update_value("attention_config", expand_attention_types(self.attention_config, self.num_layers))
assert len(self.attention_config) == self.num_layers, "Length of attention config list must equal num_layers"
for item in self.attention_config:
assert item in ATTENTION_TYPE_CHOICES, f"Attention type {item} not recognized"
if "gmlp" in self.attention_config or "amlp" in self.attention_config:
assert not self.partition_activations, "GMLP Blocks are not compatible with partition activations"
# Sparsity config
if self.sparsity_config is None:
# Can't have a default value as an empty dict so need to set it here
self.update_value("sparsity_config", {})
# Adding equal dataset weights if none are provided
if self.train_data_paths and (self.train_data_weights is None):
self.train_data_weights = [1.] * len(self.train_data_paths)
if self.valid_data_paths and (self.valid_data_weights is None):
self.valid_data_weights = [1.] * len(self.valid_data_paths)
if self.test_data_paths and (self.test_data_weights is None):
self.test_data_weights = [1.] * len(self.test_data_paths)
def get_uuid():
return shortuuid.uuid().lower()[:5]
def __init__(self, assigned=True, **kwargs):
self.id = shortuuid.uuid()[0:10]
self.created = datetime.datetime.utcnow().isoformat()
self.assigned = assigned
self.products = {}
super(Badge, self).__init__(**kwargs)
def test_incoming_message_info(self):
client = yield from connect(AMQP_URL, loop=self.loop)
queue_name = self.get_random_name("test_connection")
routing_key = self.get_random_name()
channel = yield from client.channel()
exchange = yield from channel.declare_exchange('direct',
auto_delete=True)
queue = yield from channel.declare_queue(queue_name, auto_delete=True)
yield from queue.bind(exchange, routing_key)
body = bytes(shortuuid.uuid(), 'utf-8')
info = {
'headers': {
"foo": "bar"
},
'content_type': "application/json",
'content_encoding': "text",
'delivery_mode': DeliveryMode.PERSISTENT.value,
'priority': 0,
'correlation_id': b'1',
'reply_to': 'test',
'expiration': 1.5,
'message_id': shortuuid.uuid(),
'timestamp': int(time.time()),
'type': '0',
'user_id': 'guest',
'app_id': 'test',
'body_size': len(body)
}
msg = Message(body=body,
headers={'foo': 'bar'},
content_type='application/json',
content_encoding='text',
delivery_mode=DeliveryMode.PERSISTENT,
priority=0,
correlation_id=1,
reply_to='test',
expiration=1.5,
message_id=info['message_id'],
timestamp=info['timestamp'],
type='0',
user_id='guest',
app_id='test')
yield from exchange.publish(msg, routing_key)
incoming_message = yield from queue.get(timeout=5)
incoming_message.ack()
info['synchronous'] = incoming_message.synchronous
info['routing_key'] = incoming_message.routing_key
info['redelivered'] = incoming_message.redelivered
info['exchange'] = incoming_message.exchange
info['delivery_tag'] = incoming_message.delivery_tag
info['consumer_tag'] = incoming_message.consumer_tag
info['cluster_id'] = incoming_message.cluster_id
self.assertEqual(incoming_message.body, body)
self.assertDictEqual(incoming_message.info(), info)
yield from queue.unbind(exchange, routing_key)
yield from queue.delete()
yield from wait((client.close(), client.closing), loop=self.loop)
def tmp_fname(fname=""):
"""Temporary name for a partial download"""
from shortuuid import uuid
return os.fspath(fname) + "." + uuid() + ".tmp"
def generate_uuid(cls, content):
"""Generate short uuid, used to uniquely identify logs based on content."""
return shortuuid.uuid(str(content))
def tmp_fname(fname):
""" Temporary name for a partial download """
from shortuuid import uuid
return fspath(fname) + "." + str(uuid()) + ".tmp"
def init(self) -> Union[Run, Dummy, None]: # noqa: C901
trigger.call("on_init", **self.kwargs)
s = self.settings
sweep_config = self.sweep_config
config = self.config
if s._noop:
drun = Dummy()
drun.config = wandb.wandb_sdk.wandb_config.Config()
drun.config.update(sweep_config)
drun.config.update(config)
drun.summary = DummyDict()
drun.log = lambda data, *_, **__: drun.summary.update(data)
drun.finish = lambda *_, **__: module.unset_globals()
drun.step = 0
drun.resumed = False
drun.disabled = True
drun.id = shortuuid.uuid()
drun.name = "dummy-" + drun.id
drun.dir = "/"
module.set_global(
run=drun,
config=drun.config,
log=drun.log,
summary=drun.summary,
save=drun.save,
use_artifact=drun.use_artifact,
log_artifact=drun.log_artifact,
plot_table=drun.plot_table,
alert=drun.alert,
)
return drun
if s.reinit or (s._jupyter and s.reinit is not False):
if len(self._wl._global_run_stack) > 0:
if len(self._wl._global_run_stack) > 1:
wandb.termwarn(
"If you want to track multiple runs concurrently in wandb you should use multi-processing not threads" # noqa: E501
)
last_id = self._wl._global_run_stack[-1]._run_id
jupyter = (s._jupyter and not s._silent
and ipython._get_python_type() == "jupyter")
if jupyter:
ipython.display_html(
"Finishing last run (ID:{}) before initializing another..."
.format(last_id))
self._wl._global_run_stack[-1].finish()
if jupyter:
ipython.display_html(
"...Successfully finished last run (ID:{}). Initializing new run:<br/><br/>"
.format(last_id))
elif isinstance(wandb.run, Run):
logger.info("wandb.init() called when a run is still active")
return wandb.run
use_redirect = True
stdout_master_fd, stderr_master_fd = None, None
stdout_slave_fd, stderr_slave_fd = None, None
backend = Backend()
backend.ensure_launched(
settings=s,
stdout_fd=stdout_master_fd,
stderr_fd=stderr_master_fd,
use_redirect=use_redirect,
)
backend.server_connect()
# Make sure we are logged in
# wandb_login._login(_backend=backend, _settings=self.settings)
# resuming needs access to the server, check server_status()?
run = Run(config=config, settings=s, sweep_config=sweep_config)
# Populate intial telemetry
with telemetry.context(run=run) as tel:
tel.cli_version = wandb.__version__
tel.python_version = platform.python_version()
hf_version = _huggingface_version()
if hf_version:
tel.huggingface_version = hf_version
if s._jupyter:
tel.env.jupyter = True
if s._kaggle:
tel.env.kaggle = True
if s._windows:
tel.env.windows = True
run._telemetry_imports(tel.imports_init)
run._set_console(
use_redirect=use_redirect,
stdout_slave_fd=stdout_slave_fd,
stderr_slave_fd=stderr_slave_fd,
)
run._set_library(self._wl)
run._set_backend(backend)
run._set_reporter(self._reporter)
run._set_teardown_hooks(self._teardown_hooks)
# TODO: pass mode to backend
# run_synced = None
backend._hack_set_run(run)
backend.interface.publish_header()
if s._offline:
with telemetry.context(run=run) as tel:
tel.feature.offline = True
run_proto = backend.interface._make_run(run)
backend.interface._publish_run(run_proto)
run._set_run_obj_offline(run_proto)
else:
ret = backend.interface.communicate_check_version(
current_version=wandb.__version__)
if ret:
if ret.upgrade_message:
run._set_upgraded_version_message(ret.upgrade_message)
if ret.delete_message:
run._set_deleted_version_message(ret.delete_message)
if ret.yank_message:
run._set_yanked_version_message(ret.yank_message)
run._on_init()
ret = backend.interface.communicate_run(run, timeout=30)
error_message = None
if not ret:
error_message = "Error communicating with backend"
if ret and ret.error:
error_message = ret.error.message
if error_message:
# Shutdown the backend and get rid of the logger
# we don't need to do console cleanup at this point
backend.cleanup()
self.teardown()
raise UsageError(error_message)
if ret.run.resumed:
with telemetry.context(run=run) as tel:
tel.feature.resumed = True
run._set_run_obj(ret.run)
# initiate run (stats and metadata probing)
_ = backend.interface.communicate_run_start()
self._wl._global_run_stack.append(run)
self.run = run
self.backend = backend
module.set_global(
run=run,
config=run.config,
log=run.log,
summary=run.summary,
save=run.save,
use_artifact=run.use_artifact,
log_artifact=run.log_artifact,
plot_table=run.plot_table,
alert=run.alert,
)
self._reporter.set_context(run=run)
run._on_start()
run._freeze()
return run
